add sql-datasource; port datazilla sql datasource (with tests) to treeherder

puppet/manifests/classes/python.pp

@@ -1,7 +1,7 @@
 # Install python and compiled modules for project
 class python {
     package {
-        ["python2.7-dev", "python2.7", "python-pip", "python-virtualenv"]:
+        ["python2.7-dev", "python2.7", "python-pip", "python-virtualenv", "git"]:
             ensure => installed;
     }
 
@@ -29,12 +29,13 @@ class python {
         require => Exec["create-virtualenv"],
     }
 
-    exec { "pip-install-compiled":
+    exec { "pip-install-requirements":
         command => "/home/vagrant/venv/bin/pip install \
         -r $PROJ_DIR/requirements/compiled.txt -r $PROJ_DIR/requirements/dev.txt \
         -r $PROJ_DIR/requirements/pure.txt",
         require => [
             Package['python-pip'],
+            Package['git'],
             Exec['update-distribute'],
             Exec['create-virtualenv'],
         ],

requirements/pure.txt

@@ -1,3 +1,5 @@
 oauth2==1.5.211
 South==0.7.6
 python-memcached==1.48
+
+git+git://github.com/jeads/datasource@97ce9bb710

tests/model/sql/test_models.py

@@ -0,0 +1,82 @@
+import datetime
+from contextlib import contextmanager
+
+dataset_num = 1
+
+
+def create_datasource(model, **kwargs):
+    """Utility function to easily create a test DataSource."""
+    global dataset_num
+
+    defaults = {
+        "project": "foo",
+        "dataset": dataset_num,
+        "contenttype": "jobs",
+        "host": "localhost",
+        "type": "MySQL-InnoDB",
+        "creation_date": datetime.datetime.now(),
+        #"cron_batch": "small",
+    }
+
+    dataset_num += 1
+
+    defaults.update(kwargs)
+
+    if "name" not in defaults:
+        defaults["name"] = "_".join(
+            [
+                defaults["project"],
+                defaults["contenttype"],
+                str(defaults["dataset"]),
+            ]
+        )
+
+    return model.objects.create(**defaults)
+
+
+@contextmanager
+def assert_num_queries(queries):
+    from django.db import connection
+    _old_debug_cursor = connection.use_debug_cursor
+    connection.use_debug_cursor = True
+    start_queries = len(connection.queries)
+    try:
+        yield
+        total = len(connection.queries) - start_queries
+        msg = "Expected {0} queries, executed {1}".format(queries, total)
+        assert total == queries, msg
+    finally:
+        connection.use_debug_cursor = _old_debug_cursor
+
+
+def pytest_funcarg__DataSource(request):
+    """
+    Gives a test access to the DataSource model class.
+
+    """
+    from treeherder.webapp.models import Datasource
+    return Datasource
+
+
+def test_datasources_cached(DataSource):
+    """Requesting the full list of DataSources twice only hits the DB once."""
+    create_datasource(DataSource)
+
+    DataSource.objects.cached()
+
+    with assert_num_queries(0):
+        DataSource.objects.cached()
+
+
+def test_datasource_cache_invalidated(DataSource):
+    """Saving a new datasource invalidates the datasource cache."""
+    # prime the cache
+    initial = DataSource.objects.cached()
+
+    # create a new datasource
+    create_datasource(DataSource)
+
+    print DataSource.objects.all()
+
+    # new datasource appears in the list immediately
+    assert len(DataSource.objects.cached()) == len(initial) + 1

treeherder/webapp/models.py

@@ -1,9 +1,16 @@
 from __future__ import unicode_literals
-from django.db import models
 import uuid
 import subprocess
+import os
+from django.core.cache import cache
+from django.db import models
 from treeherder import path
 
+# the cache key is specific to the database name we're pulling the data from
+SOURCES_CACHE_KEY = "treeherder-datasources"
+
+SQL_PATH = os.path.dirname(os.path.abspath(__file__))
+
 
 class Product(models.Model):
     id = models.IntegerField(primary_key=True)
@@ -141,6 +148,16 @@ class MachineNote(models.Model):
             self.id, self.machine, self.author)
 
 
+class DatasourceManager(models.Manager):
+    def cached(self):
+        """Return all datasources, caching the results."""
+        sources = cache.get(SOURCES_CACHE_KEY)
+        if not sources:
+            sources = list(self.all())
+            cache.set(SOURCES_CACHE_KEY, sources)
+        return sources
+
+
 class Datasource(models.Model):
     id = models.IntegerField(primary_key=True)
     project = models.CharField(max_length=25L)
@@ -150,11 +167,13 @@ class Datasource(models.Model):
     read_only_host = models.CharField(max_length=128L, blank=True)
     name = models.CharField(max_length=128L)
     type = models.CharField(max_length=25L)
-    oauth_consumer_key = models.CharField(max_length=45L, blank=True)
-    oauth_consumer_secret = models.CharField(max_length=45L, blank=True)
+    oauth_consumer_key = models.CharField(max_length=45L, blank=True, null=True)
+    oauth_consumer_secret = models.CharField(max_length=45L, blank=True, null=True)
     creation_date = models.DateTimeField(auto_now_add=True)
     cron_batch = models.CharField(max_length=45L, blank=True)
 
+    objects = DatasourceManager()
+
     class Meta:
         db_table = 'datasource'
         unique_together = [
@@ -162,13 +181,26 @@ class Datasource(models.Model):
             ["host", "name"],
         ]
 
+    @classmethod
+    def reset_cache(cls):
+        cache.delete(SOURCES_CACHE_KEY)
+        cls.objects.cached()
+
+    @property
+    def key(self):
+        """Unique key for a data source is the project, contenttype, dataset."""
+        return "{0} - {1} - {2}".format(
+            self.project, self.contenttype, self.dataset)
+
     def __unicode__(self):
-        return "{0} ({1})".format(
-            self.name, self.project)
+        """Unicode representation is the project's unique key."""
+        return unicode(self.key)
 
     def save(self, *args, **kwargs):
         inserting = not self.pk
-        if inserting:
+        # in case you want to add a new datasource and provide
+        # a pk, set force_insert=True when you save
+        if inserting or kwargs.get('force_insert',False):
             if not self.name:
                 self.name = "{0}_{1}_{2}".format(
                     self.project,
@@ -189,6 +221,50 @@ class Datasource(models.Model):
         if inserting:
             self.create_db()
 
+    def get_oauth_consumer_secret(self, key):
+        """
+        Return the oauth consumer secret if the key provided matches the
+        the consumer key.
+        """
+        oauth_consumer_secret = None
+        if self.oauth_consumer_key == key:
+            oauth_consumer_secret = self.oauth_consumer_secret
+        return oauth_consumer_secret
+
+    def dhub(self, procs_file_name):
+        """
+        Return a configured ``DataHub`` using the given SQL procs file.
+
+        """
+        data_source = {
+            self.key: {
+                # @@@ this should depend on self.type
+                # @@@ shouldn't have to specify this here and below
+                "hub": "MySQL",
+                "master_host": {
+                    "host": self.host,
+                    "user": settings.TREEHERDER_DATABASE_USER,
+                    "passwd": settings.TREEHERDER_DATABASE_PASSWORD,
+                    },
+                "default_db": self.name,
+                "procs": [
+                    os.path.join(SQL_PATH, procs_file_name),
+                    os.path.join(SQL_PATH, "generic.json"),
+                    ],
+                }
+            }
+
+        if self.read_only_host:
+            data_source[self.key]['read_host'] = {
+                "host": self.read_only_host,
+                "user": settings.TREEHERDER_RO_DATABASE_USER,
+                "passwd": settings.TREEHERDER_RO_DATABASE_PASSWORD,
+                }
+
+        BaseHub.add_data_source(data_source)
+        # @@@ the datahub class should depend on self.type
+        return MySQL(self.key)
+
     def create_db(self, schema_file=None):
         """
         Create the database for this source, using given SQL schema file.