diff --git a/puppet/manifests/classes/treeherder.pp b/puppet/manifests/classes/treeherder.pp index 1f9bc5c72..9cb08d626 100644 --- a/puppet/manifests/classes/treeherder.pp +++ b/puppet/manifests/classes/treeherder.pp @@ -3,4 +3,14 @@ class treeherder { package{"make": ensure => "installed" } + + package{"memcached": + ensure => "installed" + } + + service{"memcached": + ensure => running, + enable => true, + require => Package['memcached']; + } } diff --git a/puppet/manifests/vagrant.pp b/puppet/manifests/vagrant.pp index ff6d10a1a..e520a6560 100644 --- a/puppet/manifests/vagrant.pp +++ b/puppet/manifests/vagrant.pp @@ -27,6 +27,7 @@ export TREEHERDER_DATABASE_HOST='${DB_HOST}' export TREEHERDER_DATABASE_PORT='${DB_PORT}' export TREEHERDER_DEBUG='1' export TREEHERDER_DJANGO_SECRET_KEY='${DJANGO_SECRET_KEY}' +export TREEHERDER_MEMCACHED='127.0.0.1:11211' " } diff --git a/requirements/pure.txt b/requirements/pure.txt index 14362ce6f..32fc37010 100644 --- a/requirements/pure.txt +++ b/requirements/pure.txt @@ -1,2 +1,3 @@ oauth2==1.5.211 -South==0.7.6 \ No newline at end of file +South==0.7.6 +python-memcached==1.48 diff --git a/tests/conftest.py b/tests/conftest.py index d9d30de1e..3eb7b5326 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,7 @@ import os from os.path import dirname -from django.core.management import call_command import sys +from django.core.management import call_command import pytest @@ -17,6 +17,7 @@ Set DJANGO_SETTINGS_MODULE and sets up a test database. from django.conf import settings from django.test.simple import DjangoTestSuiteRunner from treeherder.webapp.models import Datasource + from django.core.cache import cache # we don't actually let Django run the tests, but we need to use some # methods of its runner for setup/teardown of dbs and some other things @@ -31,6 +32,7 @@ Set DJANGO_SETTINGS_MODULE and sets up a test database. session.django_db_config = session.django_runner.setup_databases() # init the datasource db call_command("init_master_db", interactive=False) + cache.clear() def pytest_sessionfinish(session): @@ -68,6 +70,7 @@ Roll back the Django ORM transaction and delete all the dbs created between test """ from django.test.testcases import restore_transaction_methods from django.db import transaction + from django.core.cache import cache from treeherder.webapp.models import Datasource ds_list = Datasource.objects.all() @@ -77,3 +80,4 @@ Roll back the Django ORM transaction and delete all the dbs created between test restore_transaction_methods() transaction.rollback() transaction.leave_transaction_management() + cache.clear() diff --git a/tests/test_setup.py b/tests/test_setup.py index 38fe5f49f..ed1d98aa6 100644 --- a/tests/test_setup.py +++ b/tests/test_setup.py @@ -2,6 +2,7 @@ import pytest from django.conf import settings from treeherder.webapp.models import Datasource import MySQLdb +from django.core.cache import cache @pytest.fixture @@ -42,3 +43,10 @@ def test_datasource_db_created(jobs_ds, db_conn): assert jobs_ds.name in [r[0] for r in rows], \ "When a datasource is created, a new db should be created too" db_conn.close() + + +def test_memcached_setup(): + "Test memcached is properly setup" + cache.set('my_key', 'my_value') + cache.get('my_key', 'alternative') + assert cache.get('my_key') == 'my_value' diff --git a/treeherder/cache.py b/treeherder/cache.py new file mode 100644 index 000000000..077e4a188 --- /dev/null +++ b/treeherder/cache.py @@ -0,0 +1,21 @@ +from django.core.cache.backends import memcached + + +class MemcachedCache(memcached.MemcachedCache): + """ +A subclass of Django's built-in Memcached backend that fixes some issues. + +- Allows caching forever with a timeout of 0. +- Returns the return value of set() to allow for error-checking. + +""" + def _get_memcache_timeout(self, timeout): + if timeout is None: + timeout = self.default_timeout + if not timeout: + return 0 + return super(MemcachedCache, self)._get_memcache_timeout(timeout) + + def set(self, key, value, timeout=0, version=None): + key = self.make_key(key, version=version) + return self._cache.set(key, value, self._get_memcache_timeout(timeout)) diff --git a/treeherder/settings/base.py b/treeherder/settings/base.py index e47d71ed9..b7e62d4d9 100644 --- a/treeherder/settings/base.py +++ b/treeherder/settings/base.py @@ -9,6 +9,8 @@ TREEHERDER_DATABASE_PASSWORD = os.environ.get("TREEHERDER_DATABASE_PASSWORD", "" TREEHERDER_DATABASE_HOST = os.environ.get("TREEHERDER_DATABASE_HOST", "localhost") TREEHERDER_DATABASE_PORT = os.environ.get("TREEHERDER_DATABASE_PORT", "") +TREEHERDER_MEMCACHED = os.environ.get("TREEHERDER_MEMCACHED", "") +TREEHERDER_MEMCACHED_KEY_PREFIX = os.environ.get("TREEHERDER_MEMCACHED_KEY_PREFIX", "treeherder") DEBUG = os.environ.get("TREEHERDER_DEBUG", False) @@ -120,3 +122,15 @@ DATABASES = { "PORT" : TREEHERDER_DATABASE_PORT, } } + +CACHES = { + "default": { + "BACKEND": "treeherder.cache.MemcachedCache", + "LOCATION": TREEHERDER_MEMCACHED, + "TIMEOUT": 0, + # bumping this is effectively equivalent to restarting memcached + "VERSION": 1, + } +} + +KEY_PREFIX = TREEHERDER_MEMCACHED_KEY_PREFIX diff --git a/treeherder/settings/local.sample.py b/treeherder/settings/local.sample.py index fa26ac004..172d80ed6 100644 --- a/treeherder/settings/local.sample.py +++ b/treeherder/settings/local.sample.py @@ -6,6 +6,9 @@ TREEHERDER_DATABASE_PASSWORD = os.environ.get("TREEHERDER_DATABASE_PASSWORD", "" TREEHERDER_DATABASE_HOST = os.environ.get("TREEHERDER_DATABASE_HOST", "localhost") TREEHERDER_DATABASE_PORT = os.environ.get("TREEHERDER_DATABASE_PORT", "") +TREEHERDER_MEMCACHED = os.environ.get("TREEHERDER_MEMCACHED", "") +TREEHERDER_MEMCACHED_KEY_PREFIX = os.environ.get("TREEHERDER_MEMCACHED_KEY_PREFIX", "treeherder") + # Applications useful for development, e.g. debug_toolbar, django_extensions. # Always empty in production LOCAL_APPS = []