From 5b8063cef83213d0a224f380b3853bc9c46bf8ce Mon Sep 17 00:00:00 2001 From: mdoglio Date: Tue, 26 Mar 2013 16:24:38 +0000 Subject: [PATCH 1/4] add memcached setup --- puppet/manifests/classes/treeherder.pp | 10 ++++++++++ puppet/manifests/vagrant.pp | 1 + requirements/pure.txt | 3 ++- tests/conftest.py | 6 +++++- tests/test_setup.py | 8 ++++++++ treeherder/cache.py | 21 +++++++++++++++++++++ treeherder/settings/base.py | 14 ++++++++++++++ treeherder/settings/local.sample.py | 3 +++ 8 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 treeherder/cache.py diff --git a/puppet/manifests/classes/treeherder.pp b/puppet/manifests/classes/treeherder.pp index 1f9bc5c72..9cb08d626 100644 --- a/puppet/manifests/classes/treeherder.pp +++ b/puppet/manifests/classes/treeherder.pp @@ -3,4 +3,14 @@ class treeherder { package{"make": ensure => "installed" } + + package{"memcached": + ensure => "installed" + } + + service{"memcached": + ensure => running, + enable => true, + require => Package['memcached']; + } } diff --git a/puppet/manifests/vagrant.pp b/puppet/manifests/vagrant.pp index ff6d10a1a..e520a6560 100644 --- a/puppet/manifests/vagrant.pp +++ b/puppet/manifests/vagrant.pp @@ -27,6 +27,7 @@ export TREEHERDER_DATABASE_HOST='${DB_HOST}' export TREEHERDER_DATABASE_PORT='${DB_PORT}' export TREEHERDER_DEBUG='1' export TREEHERDER_DJANGO_SECRET_KEY='${DJANGO_SECRET_KEY}' +export TREEHERDER_MEMCACHED='127.0.0.1:11211' " } diff --git a/requirements/pure.txt b/requirements/pure.txt index 14362ce6f..32fc37010 100644 --- a/requirements/pure.txt +++ b/requirements/pure.txt @@ -1,2 +1,3 @@ oauth2==1.5.211 -South==0.7.6 \ No newline at end of file +South==0.7.6 +python-memcached==1.48 diff --git a/tests/conftest.py b/tests/conftest.py index d9d30de1e..3eb7b5326 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,7 @@ import os from os.path import dirname -from django.core.management import call_command import sys +from django.core.management import call_command import pytest @@ -17,6 +17,7 @@ Set DJANGO_SETTINGS_MODULE and sets up a test database. from django.conf import settings from django.test.simple import DjangoTestSuiteRunner from treeherder.webapp.models import Datasource + from django.core.cache import cache # we don't actually let Django run the tests, but we need to use some # methods of its runner for setup/teardown of dbs and some other things @@ -31,6 +32,7 @@ Set DJANGO_SETTINGS_MODULE and sets up a test database. session.django_db_config = session.django_runner.setup_databases() # init the datasource db call_command("init_master_db", interactive=False) + cache.clear() def pytest_sessionfinish(session): @@ -68,6 +70,7 @@ Roll back the Django ORM transaction and delete all the dbs created between test """ from django.test.testcases import restore_transaction_methods from django.db import transaction + from django.core.cache import cache from treeherder.webapp.models import Datasource ds_list = Datasource.objects.all() @@ -77,3 +80,4 @@ Roll back the Django ORM transaction and delete all the dbs created between test restore_transaction_methods() transaction.rollback() transaction.leave_transaction_management() + cache.clear() diff --git a/tests/test_setup.py b/tests/test_setup.py index 38fe5f49f..ed1d98aa6 100644 --- a/tests/test_setup.py +++ b/tests/test_setup.py @@ -2,6 +2,7 @@ import pytest from django.conf import settings from treeherder.webapp.models import Datasource import MySQLdb +from django.core.cache import cache @pytest.fixture @@ -42,3 +43,10 @@ def test_datasource_db_created(jobs_ds, db_conn): assert jobs_ds.name in [r[0] for r in rows], \ "When a datasource is created, a new db should be created too" db_conn.close() + + +def test_memcached_setup(): + "Test memcached is properly setup" + cache.set('my_key', 'my_value') + cache.get('my_key', 'alternative') + assert cache.get('my_key') == 'my_value' diff --git a/treeherder/cache.py b/treeherder/cache.py new file mode 100644 index 000000000..077e4a188 --- /dev/null +++ b/treeherder/cache.py @@ -0,0 +1,21 @@ +from django.core.cache.backends import memcached + + +class MemcachedCache(memcached.MemcachedCache): + """ +A subclass of Django's built-in Memcached backend that fixes some issues. + +- Allows caching forever with a timeout of 0. +- Returns the return value of set() to allow for error-checking. + +""" + def _get_memcache_timeout(self, timeout): + if timeout is None: + timeout = self.default_timeout + if not timeout: + return 0 + return super(MemcachedCache, self)._get_memcache_timeout(timeout) + + def set(self, key, value, timeout=0, version=None): + key = self.make_key(key, version=version) + return self._cache.set(key, value, self._get_memcache_timeout(timeout)) diff --git a/treeherder/settings/base.py b/treeherder/settings/base.py index e47d71ed9..b7e62d4d9 100644 --- a/treeherder/settings/base.py +++ b/treeherder/settings/base.py @@ -9,6 +9,8 @@ TREEHERDER_DATABASE_PASSWORD = os.environ.get("TREEHERDER_DATABASE_PASSWORD", "" TREEHERDER_DATABASE_HOST = os.environ.get("TREEHERDER_DATABASE_HOST", "localhost") TREEHERDER_DATABASE_PORT = os.environ.get("TREEHERDER_DATABASE_PORT", "") +TREEHERDER_MEMCACHED = os.environ.get("TREEHERDER_MEMCACHED", "") +TREEHERDER_MEMCACHED_KEY_PREFIX = os.environ.get("TREEHERDER_MEMCACHED_KEY_PREFIX", "treeherder") DEBUG = os.environ.get("TREEHERDER_DEBUG", False) @@ -120,3 +122,15 @@ DATABASES = { "PORT" : TREEHERDER_DATABASE_PORT, } } + +CACHES = { + "default": { + "BACKEND": "treeherder.cache.MemcachedCache", + "LOCATION": TREEHERDER_MEMCACHED, + "TIMEOUT": 0, + # bumping this is effectively equivalent to restarting memcached + "VERSION": 1, + } +} + +KEY_PREFIX = TREEHERDER_MEMCACHED_KEY_PREFIX diff --git a/treeherder/settings/local.sample.py b/treeherder/settings/local.sample.py index fa26ac004..172d80ed6 100644 --- a/treeherder/settings/local.sample.py +++ b/treeherder/settings/local.sample.py @@ -6,6 +6,9 @@ TREEHERDER_DATABASE_PASSWORD = os.environ.get("TREEHERDER_DATABASE_PASSWORD", "" TREEHERDER_DATABASE_HOST = os.environ.get("TREEHERDER_DATABASE_HOST", "localhost") TREEHERDER_DATABASE_PORT = os.environ.get("TREEHERDER_DATABASE_PORT", "") +TREEHERDER_MEMCACHED = os.environ.get("TREEHERDER_MEMCACHED", "") +TREEHERDER_MEMCACHED_KEY_PREFIX = os.environ.get("TREEHERDER_MEMCACHED_KEY_PREFIX", "treeherder") + # Applications useful for development, e.g. debug_toolbar, django_extensions. # Always empty in production LOCAL_APPS = [] From 8546748bc52f53944f8e0da87d402d9bfa89429d Mon Sep 17 00:00:00 2001 From: mdoglio Date: Thu, 28 Mar 2013 00:40:42 +0000 Subject: [PATCH 2/4] tests use cache versioning to keep isolation --- tests/conftest.py | 18 ++++++++++++++++++ tests/test_setup.py | 6 +++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3eb7b5326..1e2c05ec5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -30,6 +30,9 @@ Set DJANGO_SETTINGS_MODULE and sets up a test database. settings.DATABASES["default"]["TEST_NAME"] = "{0}test_treeherder".format(prefix) # this sets up a clean test-only database session.django_db_config = session.django_runner.setup_databases() + + increment_cache_key_prefix() + # init the datasource db call_command("init_master_db", interactive=False) cache.clear() @@ -60,6 +63,8 @@ providing test isolation. transaction.managed(True) disable_transaction_methods() + increment_cache_key_prefix() + def pytest_runtest_teardown(item): """ @@ -81,3 +86,16 @@ Roll back the Django ORM transaction and delete all the dbs created between test transaction.rollback() transaction.leave_transaction_management() cache.clear() + + +def increment_cache_key_prefix(): + """Increment a cache prefix to effectively clear the cache.""" + from django.core.cache import cache + cache.key_prefix = "" + prefix_counter_cache_key = "treeherder-tests-key-prefix-counter" + try: + key_prefix_counter = cache.incr(prefix_counter_cache_key) + except ValueError: + key_prefix_counter = 0 + cache.set(prefix_counter_cache_key, key_prefix_counter) + cache.key_prefix = "t{0}".format(key_prefix_counter) diff --git a/tests/test_setup.py b/tests/test_setup.py index ed1d98aa6..f06ca67cf 100644 --- a/tests/test_setup.py +++ b/tests/test_setup.py @@ -47,6 +47,6 @@ def test_datasource_db_created(jobs_ds, db_conn): def test_memcached_setup(): "Test memcached is properly setup" - cache.set('my_key', 'my_value') - cache.get('my_key', 'alternative') - assert cache.get('my_key') == 'my_value' + k, v = 'my_key', 'my_value' + cache.set(k, v) + assert cache.get(k) == v From ac44d1bc10e10f9b69c6e9b88d859ca0f7fa9678 Mon Sep 17 00:00:00 2001 From: mdoglio Date: Thu, 28 Mar 2013 00:45:34 +0000 Subject: [PATCH 3/4] remove cache.clear() from tests setup/teardown --- tests/conftest.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 1e2c05ec5..c978b9ee8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -35,7 +35,6 @@ Set DJANGO_SETTINGS_MODULE and sets up a test database. # init the datasource db call_command("init_master_db", interactive=False) - cache.clear() def pytest_sessionfinish(session): @@ -85,7 +84,6 @@ Roll back the Django ORM transaction and delete all the dbs created between test restore_transaction_methods() transaction.rollback() transaction.leave_transaction_management() - cache.clear() def increment_cache_key_prefix(): From 41116af5bb56fc0521b8e179c8f4a0aad685cf6d Mon Sep 17 00:00:00 2001 From: mdoglio Date: Thu, 28 Mar 2013 00:49:11 +0000 Subject: [PATCH 4/4] remove unused import --- tests/conftest.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index c978b9ee8..d9d0c3b75 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,7 +17,6 @@ Set DJANGO_SETTINGS_MODULE and sets up a test database. from django.conf import settings from django.test.simple import DjangoTestSuiteRunner from treeherder.webapp.models import Datasource - from django.core.cache import cache # we don't actually let Django run the tests, but we need to use some # methods of its runner for setup/teardown of dbs and some other things @@ -39,8 +38,6 @@ Set DJANGO_SETTINGS_MODULE and sets up a test database. def pytest_sessionfinish(session): """Tear down the test environment, including databases.""" - from treeherder.webapp.models import Datasource - session.django_runner.teardown_databases(session.django_db_config) session.django_runner.teardown_test_environment() @@ -74,7 +71,6 @@ Roll back the Django ORM transaction and delete all the dbs created between test """ from django.test.testcases import restore_transaction_methods from django.db import transaction - from django.core.cache import cache from treeherder.webapp.models import Datasource ds_list = Datasource.objects.all()