Merge pull request #11 from mozilla/add-memcached-setup

add memcached setup

puppet/manifests/classes/treeherder.pp

@@ -3,4 +3,14 @@ class treeherder {
     package{"make":
         ensure => "installed"
     }
+
+    package{"memcached":
+        ensure => "installed"
+    }
+
+    service{"memcached":
+        ensure => running,
+        enable => true,
+        require => Package['memcached'];
+    }
 }

puppet/manifests/vagrant.pp

@@ -27,6 +27,7 @@ export TREEHERDER_DATABASE_HOST='${DB_HOST}'
 export TREEHERDER_DATABASE_PORT='${DB_PORT}'
 export TREEHERDER_DEBUG='1'
 export TREEHERDER_DJANGO_SECRET_KEY='${DJANGO_SECRET_KEY}'
+export TREEHERDER_MEMCACHED='127.0.0.1:11211'
 "
 }
 

requirements/pure.txt

@@ -1,2 +1,3 @@
 oauth2==1.5.211
 South==0.7.6
+python-memcached==1.48

tests/conftest.py

@@ -1,7 +1,7 @@
 import os
 from os.path import dirname
-from django.core.management import call_command
 import sys
+from django.core.management import call_command
 import pytest
 
 
@@ -29,14 +29,15 @@ Set DJANGO_SETTINGS_MODULE and sets up a test database.
     settings.DATABASES["default"]["TEST_NAME"] = "{0}test_treeherder".format(prefix)
     # this sets up a clean test-only database
     session.django_db_config = session.django_runner.setup_databases()
+
+    increment_cache_key_prefix()
+
     # init the datasource db
     call_command("init_master_db", interactive=False)
 
 
 def pytest_sessionfinish(session):
     """Tear down the test environment, including databases."""
-    from treeherder.webapp.models import Datasource
-
     session.django_runner.teardown_databases(session.django_db_config)
     session.django_runner.teardown_test_environment()
 
@@ -58,6 +59,8 @@ providing test isolation.
     transaction.managed(True)
     disable_transaction_methods()
 
+    increment_cache_key_prefix()
+
 
 def pytest_runtest_teardown(item):
     """
@@ -77,3 +80,16 @@ Roll back the Django ORM transaction and delete all the dbs created between test
     restore_transaction_methods()
     transaction.rollback()
     transaction.leave_transaction_management()
+
+
+def increment_cache_key_prefix():
+    """Increment a cache prefix to effectively clear the cache."""
+    from django.core.cache import cache
+    cache.key_prefix = ""
+    prefix_counter_cache_key = "treeherder-tests-key-prefix-counter"
+    try:
+        key_prefix_counter = cache.incr(prefix_counter_cache_key)
+    except ValueError:
+        key_prefix_counter = 0
+        cache.set(prefix_counter_cache_key, key_prefix_counter)
+    cache.key_prefix = "t{0}".format(key_prefix_counter)

tests/test_setup.py

@@ -2,6 +2,7 @@ import pytest
 from django.conf import settings
 from treeherder.webapp.models import Datasource
 import MySQLdb
+from django.core.cache import cache
 
 
 @pytest.fixture
@@ -42,3 +43,10 @@ def test_datasource_db_created(jobs_ds, db_conn):
     assert jobs_ds.name in [r[0] for r in rows], \
         "When a datasource is created, a new db should be created too"
     db_conn.close()
+
+
+def test_memcached_setup():
+    "Test memcached is properly setup"
+    k, v = 'my_key', 'my_value'
+    cache.set(k, v)
+    assert cache.get(k) == v

treeherder/cache.py

@@ -0,0 +1,21 @@
+from django.core.cache.backends import memcached
+
+
+class MemcachedCache(memcached.MemcachedCache):
+    """
+A subclass of Django's built-in Memcached backend that fixes some issues.
+
+- Allows caching forever with a timeout of 0.
+- Returns the return value of set() to allow for error-checking.
+
+"""
+    def _get_memcache_timeout(self, timeout):
+        if timeout is None:
+            timeout = self.default_timeout
+        if not timeout:
+            return 0
+        return super(MemcachedCache, self)._get_memcache_timeout(timeout)
+
+    def set(self, key, value, timeout=0, version=None):
+        key = self.make_key(key, version=version)
+        return self._cache.set(key, value, self._get_memcache_timeout(timeout))

treeherder/settings/base.py

@@ -9,6 +9,8 @@ TREEHERDER_DATABASE_PASSWORD = os.environ.get("TREEHERDER_DATABASE_PASSWORD", ""
 TREEHERDER_DATABASE_HOST     = os.environ.get("TREEHERDER_DATABASE_HOST", "localhost")
 TREEHERDER_DATABASE_PORT     = os.environ.get("TREEHERDER_DATABASE_PORT", "")
 
+TREEHERDER_MEMCACHED = os.environ.get("TREEHERDER_MEMCACHED", "")
+TREEHERDER_MEMCACHED_KEY_PREFIX = os.environ.get("TREEHERDER_MEMCACHED_KEY_PREFIX", "treeherder")
 DEBUG = os.environ.get("TREEHERDER_DEBUG", False)
 
 
@@ -120,3 +122,15 @@ DATABASES = {
         "PORT"     : TREEHERDER_DATABASE_PORT,
     }
 }
+
+CACHES = {
+    "default": {
+        "BACKEND": "treeherder.cache.MemcachedCache",
+        "LOCATION": TREEHERDER_MEMCACHED,
+        "TIMEOUT": 0,
+        # bumping this is effectively equivalent to restarting memcached
+        "VERSION": 1,
+        }
+}
+
+KEY_PREFIX = TREEHERDER_MEMCACHED_KEY_PREFIX

treeherder/settings/local.sample.py

@@ -6,6 +6,9 @@ TREEHERDER_DATABASE_PASSWORD = os.environ.get("TREEHERDER_DATABASE_PASSWORD", ""
 TREEHERDER_DATABASE_HOST     = os.environ.get("TREEHERDER_DATABASE_HOST", "localhost")
 TREEHERDER_DATABASE_PORT     = os.environ.get("TREEHERDER_DATABASE_PORT", "")
 
+TREEHERDER_MEMCACHED = os.environ.get("TREEHERDER_MEMCACHED", "")
+TREEHERDER_MEMCACHED_KEY_PREFIX = os.environ.get("TREEHERDER_MEMCACHED_KEY_PREFIX", "treeherder")
+
 # Applications useful for development, e.g. debug_toolbar, django_extensions.
 # Always empty in production
 LOCAL_APPS = []