add memcached setup

puppet/manifests/classes/treeherder.pp

@@ -3,4 +3,14 @@ class treeherder {
     package{"make":
         ensure => "installed"
     }
+
+    package{"memcached":
+        ensure => "installed"
+    }
+
+    service{"memcached":
+        ensure => running,
+        enable => true,
+        require => Package['memcached'];
+    }
 }

puppet/manifests/vagrant.pp

@@ -27,6 +27,7 @@ export TREEHERDER_DATABASE_HOST='${DB_HOST}'
 export TREEHERDER_DATABASE_PORT='${DB_PORT}'
 export TREEHERDER_DEBUG='1'
 export TREEHERDER_DJANGO_SECRET_KEY='${DJANGO_SECRET_KEY}'
+export TREEHERDER_MEMCACHED='127.0.0.1:11211'
 "
 }
 

requirements/pure.txt

@@ -1,2 +1,3 @@
 oauth2==1.5.211
-South==0.7.6
+South==0.7.6
+python-memcached==1.48

tests/conftest.py

@@ -1,7 +1,7 @@
 import os
 from os.path import dirname
-from django.core.management import call_command
 import sys
+from django.core.management import call_command
 import pytest
 
 
@@ -17,6 +17,7 @@ Set DJANGO_SETTINGS_MODULE and sets up a test database.
     from django.conf import settings
     from django.test.simple import DjangoTestSuiteRunner
     from treeherder.webapp.models import Datasource
+    from django.core.cache import cache
 
     # we don't actually let Django run the tests, but we need to use some
     # methods of its runner for setup/teardown of dbs and some other things
@@ -31,6 +32,7 @@ Set DJANGO_SETTINGS_MODULE and sets up a test database.
     session.django_db_config = session.django_runner.setup_databases()
     # init the datasource db
     call_command("init_master_db", interactive=False)
+    cache.clear()
 
 
 def pytest_sessionfinish(session):
@@ -68,6 +70,7 @@ Roll back the Django ORM transaction and delete all the dbs created between test
 """
     from django.test.testcases import restore_transaction_methods
     from django.db import transaction
+    from django.core.cache import cache
     from treeherder.webapp.models import Datasource
 
     ds_list = Datasource.objects.all()
@@ -77,3 +80,4 @@ Roll back the Django ORM transaction and delete all the dbs created between test
     restore_transaction_methods()
     transaction.rollback()
     transaction.leave_transaction_management()
+    cache.clear()

tests/test_setup.py

@@ -2,6 +2,7 @@ import pytest
 from django.conf import settings
 from treeherder.webapp.models import Datasource
 import MySQLdb
+from django.core.cache import cache
 
 
 @pytest.fixture
@@ -42,3 +43,10 @@ def test_datasource_db_created(jobs_ds, db_conn):
     assert jobs_ds.name in [r[0] for r in rows], \
         "When a datasource is created, a new db should be created too"
     db_conn.close()
+
+
+def test_memcached_setup():
+    "Test memcached is properly setup"
+    cache.set('my_key', 'my_value')
+    cache.get('my_key', 'alternative')
+    assert cache.get('my_key') == 'my_value'

treeherder/cache.py

@@ -0,0 +1,21 @@
+from django.core.cache.backends import memcached
+
+
+class MemcachedCache(memcached.MemcachedCache):
+    """
+A subclass of Django's built-in Memcached backend that fixes some issues.
+
+- Allows caching forever with a timeout of 0.
+- Returns the return value of set() to allow for error-checking.
+
+"""
+    def _get_memcache_timeout(self, timeout):
+        if timeout is None:
+            timeout = self.default_timeout
+        if not timeout:
+            return 0
+        return super(MemcachedCache, self)._get_memcache_timeout(timeout)
+
+    def set(self, key, value, timeout=0, version=None):
+        key = self.make_key(key, version=version)
+        return self._cache.set(key, value, self._get_memcache_timeout(timeout))

treeherder/settings/base.py

@@ -9,6 +9,8 @@ TREEHERDER_DATABASE_PASSWORD = os.environ.get("TREEHERDER_DATABASE_PASSWORD", ""
 TREEHERDER_DATABASE_HOST     = os.environ.get("TREEHERDER_DATABASE_HOST", "localhost")
 TREEHERDER_DATABASE_PORT     = os.environ.get("TREEHERDER_DATABASE_PORT", "")
 
+TREEHERDER_MEMCACHED = os.environ.get("TREEHERDER_MEMCACHED", "")
+TREEHERDER_MEMCACHED_KEY_PREFIX = os.environ.get("TREEHERDER_MEMCACHED_KEY_PREFIX", "treeherder")
 DEBUG = os.environ.get("TREEHERDER_DEBUG", False)
 
 
@@ -120,3 +122,15 @@ DATABASES = {
         "PORT"     : TREEHERDER_DATABASE_PORT,
     }
 }
+
+CACHES = {
+    "default": {
+        "BACKEND": "treeherder.cache.MemcachedCache",
+        "LOCATION": TREEHERDER_MEMCACHED,
+        "TIMEOUT": 0,
+        # bumping this is effectively equivalent to restarting memcached
+        "VERSION": 1,
+        }
+}
+
+KEY_PREFIX = TREEHERDER_MEMCACHED_KEY_PREFIX

treeherder/settings/local.sample.py

@@ -6,6 +6,9 @@ TREEHERDER_DATABASE_PASSWORD = os.environ.get("TREEHERDER_DATABASE_PASSWORD", ""
 TREEHERDER_DATABASE_HOST     = os.environ.get("TREEHERDER_DATABASE_HOST", "localhost")
 TREEHERDER_DATABASE_PORT     = os.environ.get("TREEHERDER_DATABASE_PORT", "")
 
+TREEHERDER_MEMCACHED = os.environ.get("TREEHERDER_MEMCACHED", "")
+TREEHERDER_MEMCACHED_KEY_PREFIX = os.environ.get("TREEHERDER_MEMCACHED_KEY_PREFIX", "treeherder")
+
 # Applications useful for development, e.g. debug_toolbar, django_extensions.
 # Always empty in production
 LOCAL_APPS = []