Remove the need for a settings_local.py (bug 1050155)

This commit is contained in:
Mathieu Agopian 2014-08-08 14:08:46 +02:00
Родитель 98088b9c7a
Коммит a1e1c8c5fb
37 изменённых файлов: 253 добавлений и 622 удалений

1
.gitignore поставляемый
Просмотреть файл

@ -1,6 +1,7 @@
.env
settings_local.py*
settings_local_*.py
local_settings.py
shellng_local.py
*.py[co]
*.sw[po]

Просмотреть файл

@ -1,8 +1,7 @@
language: python
python:
- "2.7"
service:
- redis-server
services:
- memcached
before_install:
- wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-0.90.13.deb && sudo dpkg -i elasticsearch-0.90.13.deb
@ -12,14 +11,12 @@ install:
- pip install --no-deps -r requirements/test.txt --find-links https://pyrepo.addons.mozilla.org/
- pip install --no-deps -r requirements/compiled.txt --find-links https://pyrepo.addons.mozilla.org/
- npm install
- export PATH="./node_modules/.bin/:${PATH}"
before_script:
- mysql -e 'create database olympia;'
- echo -e "from lib.settings_base import *\nCELERY_ALWAYS_EAGER = True\nRUNNING_IN_JENKINS = True" > settings_local.py
- make update_submodules
- make update_assets
script:
- make test
- DJANGO_SETTINGS_MODULE=settings_ci make test
notifications:
irc:
channels:

Просмотреть файл

@ -1,8 +1,3 @@
# You can set these variables from the command line.
DJANGO=python manage.py
SETTINGS=settings_local
RUN_ES_TESTS=1
.PHONY: help docs test test_force_db tdd test_failed update_code update_submodules update_deps update_db update_assets update_landfill full_update reindex
help:
@ -20,23 +15,21 @@ help:
@echo " update_landfill to load the landfill database data"
@echo " reindex to reindex everything in elasticsearch, for AMO"
@echo "Check the Makefile to know exactly what each target is doing. If you see a "
@echo "target using something like $(SETTINGS), you can make it use another value:"
@echo " make SETTINGS=settings_mine docs"
docs:
$(MAKE) -C docs html
test:
RUN_ES_TESTS=$(RUN_ES_TESTS) $(DJANGO) test --settings=$(SETTINGS) --with-blockage --noinput --logging-clear-handlers --with-id -v 2 $(ARGS)
python manage.py test --with-blockage --noinput --logging-clear-handlers --with-id -v 2 $(ARGS)
test_force_db:
RUN_ES_TESTS=$(RUN_ES_TESTS) FORCE_DB=1 $(DJANGO) test --settings=$(SETTINGS) --with-blockage --noinput --logging-clear-handlers --with-id -v 2 $(ARGS)
FORCE_DB=1 python manage.py test --with-blockage --noinput --logging-clear-handlers --with-id -v 2 $(ARGS)
tdd:
RUN_ES_TESTS=$(RUN_ES_TESTS) $(DJANGO) test --settings=$(SETTINGS) --with-blockage --noinput --failfast --pdb --with-id -v 2 $(ARGS)
python manage.py test --with-blockage --noinput --failfast --pdb --with-id -v 2 $(ARGS)
test_failed:
RUN_ES_TESTS=$(RUN_ES_TESTS) $(DJANGO) test --settings=$(SETTINGS) --with-blockage --noinput --logging-clear-handlers --with-id -v 2 --failed $(ARGS)
python manage.py test --with-blockage --noinput --logging-clear-handlers --with-id -v 2 --failed $(ARGS)
update_code:
git checkout master && git pull
@ -54,13 +47,13 @@ update_db:
schematic migrations
update_assets:
$(DJANGO) compress_assets --settings=$(SETTINGS)
$(DJANGO) collectstatic --settings=$(SETTINGS) --noinput
python manage.py compress_assets
python manage.py collectstatic --noinput
full_update: update_code update_deps update_db update_assets
update_landfill:
$(DJANGO) install_landfill --settings=$(SETTINGS) $(ARGS)
python manage.py install_landfill $(ARGS)
reindex:
$(DJANGO) reindex --settings=$(SETTINGS) $(ARGS)
python manage.py reindex $(ARGS)

Просмотреть файл

@ -144,9 +144,9 @@ class TestThemeUpdate(amo.tests.TestCase):
up.get_update()
image_path = up.image_path('foo.png')
# This is ugly. It's needed because services.theme_update imports
# settings_local, and settings_test is overriding MEDIA_ROOT.
import settings_local
with self.settings(MEDIA_ROOT=settings_local.MEDIA_ROOT):
# settings, and settings_test is overriding MEDIA_ROOT.
import settings
with self.settings(MEDIA_ROOT=settings.MEDIA_ROOT):
assert user_media_path('addons') in image_path
def test_image_url(self):

Просмотреть файл

@ -14,7 +14,6 @@ from amo.helpers import user_media_url
from applications.models import Application, AppVersion
from files.models import File
from services import update
import settings_local
from versions.models import ApplicationsVersions, Version
@ -589,14 +588,6 @@ class TestResponse(VersionCheckMixin, amo.tests.TestCase):
self.mac = amo.PLATFORM_MAC
self.win = amo.PLATFORM_WIN
self.old_debug = settings_local.DEBUG
settings_local.DEBUG = False
def tearDown(self):
settings_local.DEBUG = self.old_debug
super(TestResponse, self).tearDown()
def test_bad_guid(self):
data = self.good_data.copy()
data["id"] = "garbage"

Просмотреть файл

@ -906,8 +906,7 @@ class TestCollectionSearch(SearchBase):
eq_(doc('.listing-footer').length, 0)
def test_results_name_query(self):
if getattr(settings, 'RUNNING_IN_JENKINS', False):
raise SkipTest('Passes locally but fails on Jenkins :(')
raise SkipTest('Fails randomly (bug 1050754)')
self._generate()

Просмотреть файл

@ -165,7 +165,7 @@ def show_settings(request):
for i in settings.HERA:
settings_dict['HERA'].append(debug.cleanse_setting('HERA', i))
# Retain this so that legacy PAYPAL_CGI_AUTH variables in settings_local
# Retain this so that legacy PAYPAL_CGI_AUTH variables in local settings
# are not exposed.
for i in ['PAYPAL_EMBEDDED_AUTH', 'PAYPAL_CGI_AUTH',
'GOOGLE_ANALYTICS_CREDENTIALS']:

Просмотреть файл

@ -0,0 +1,19 @@
from settings import * # noqa
# To activate the Django debug toolbar.
INSTALLED_APPS += (
'debug_toolbar',
'fixture_magic',
)
DEBUG_TOOLBAR_PATCH_SETTINGS = False # Prevent DDT from patching the settings.
INTERNAL_IPS = ('127.0.0.1',)
MIDDLEWARE_CLASSES += ('debug_toolbar.middleware.DebugToolbarMiddleware',)
# Use the real stuff.
CACHES = {
'default': {
'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
'LOCATION': 'localhost:11211',
}
}

Просмотреть файл

@ -1,94 +0,0 @@
from default.settings import *
DEBUG = True
TEMPLATE_DEBUG = DEBUG
DEBUG_PROPAGATE_EXCEPTIONS = DEBUG
# These apps are great during development.
INSTALLED_APPS += (
'debug_toolbar',
'django.contrib.staticfiles', # Needed by the latest versions of DDT.
'django_extensions',
'fixture_magic',
)
# Prevent DDT from patching the settings automatically: this would result in
# circular imports for us. See DDT docs for more information.
DEBUG_TOOLBAR_PATCH_SETTINGS = False
# You want one of the caching backends. Dummy won't do any caching, locmem is
# cleared every time you restart the server, and memcached is what we run in
# production.
#CACHES = {
# 'default': {
# 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
# 'LOCATION': 'localhost:11211',
# }
#}
# Here we use the LocMemCache backend from cache-machine, as it interprets the
# "0" timeout parameter of ``cache`` in the same way as the Memcached backend:
# as infinity. Django's LocMemCache backend interprets it as a "0 seconds"
# timeout (and thus doesn't cache at all).
CACHES = {
'default': {
'BACKEND': 'caching.backends.locmem.LocMemCache',
'LOCATION': 'olympia',
}
}
# Caching is required for CSRF to work, please do not use the dummy cache.
DATABASES = {
'default': {
'NAME': 'olympia',
'ENGINE': 'django.db.backends.mysql',
'USER': 'root',
'PASSWORD': '',
'OPTIONS': {'init_command': 'SET storage_engine=InnoDB'},
'TEST_CHARSET': 'utf8',
'TEST_COLLATION': 'utf8_general_ci',
},
}
# Skip indexing ES to speed things up?
SKIP_SEARCH_INDEX = False
LOG_LEVEL = logging.DEBUG
HAS_SYSLOG = False
# For debug toolbar.
if DEBUG:
INTERNAL_IPS = ('127.0.0.1',)
MIDDLEWARE_CLASSES += ('debug_toolbar.middleware.DebugToolbarMiddleware',)
# If you're not running on SSL you'll want this to be False.
SESSION_COOKIE_SECURE = False
SESSION_COOKIE_DOMAIN = None
# Run tasks immediately, don't try using the queue.
CELERY_ALWAYS_EAGER = True
# Disables custom routing in settings.py so that tasks actually run.
CELERY_ROUTES = {}
# Disable timeout code during development because it uses the signal module
# which can only run in the main thread. Celery uses threads in dev.
VALIDATOR_TIMEOUT = -1
# The user id to use when logging in tasks. You should set this to a user that
# exists in your site.
# TASK_USER_ID = 1
# If you want to allow self-reviews for add-ons/apps, then enable this.
# In production we do not want to allow this.
ALLOW_SELF_REVIEWS = True
# If you want to skip pre-generation locally, disable it:
PRE_GENERATE_APKS = False
# Assuming you did `npm install` (and not `-g`) like you were supposed to,
# this will be the path to the `stylus` and `lessc` executables.
STYLUS_BIN = path('node_modules/stylus/bin/stylus')
LESS_BIN = path('node_modules/less/bin/lessc')
# Locally we typically don't run more than 1 elasticsearch node. So we set
# replicas to zero.
ES_DEFAULT_NUM_REPLICAS = 0

Просмотреть файл

@ -1,70 +0,0 @@
from settings import *
DEBUG = False
TEMPLATE_DEBUG = False
# The default database should point to the master.
DATABASES = {
'default': {
'NAME': 'olympia',
'ENGINE': 'django.db.backends.mysql',
'HOST': '',
'PORT': '',
'USER': '',
'PASSWORD': '',
'OPTIONS': {'init_command': 'SET storage_engine=InnoDB'},
},
'slave': {
'NAME': 'olympia',
'ENGINE': 'django.db.backends.mysql',
'HOST': '',
'PORT': '',
'USER': '',
'PASSWORD': '',
'OPTIONS': {'init_command': 'SET storage_engine=InnoDB'},
},
}
# Put the aliases for slave databases in this list.
SLAVE_DATABASES = ['slave']
# Use IP:PORT pairs separated by semicolons.
CACHES = {
'default': {
'BACKEND': 'caching.backends.memcached.MemcachedCache',
'LOCATION': ['localhost:11211', 'localhost:11212'],
'TIMEOUT': 500,
}
}
# This is used to hash some things in Django.
SECRET_KEY = 'replace me with something long'
LOG_LEVEL = logging.WARNING
DEBUG_PROPAGATE_EXCEPTIONS = DEBUG
# Sample heka configuration. Uncommented, this would override what is in
# lib/settings_base.py.
# HEKA_CONF = {
# 'logger': 'olympia',
# 'stream': {
# 'class': 'heka.streams.UdpStream',
# 'host': ['10.0.1.5', '10.0.1.10']
# 'port': 5566
# },
# 'plugins': {
# 'raven': ('heka_raven.raven_plugin.config_plugin',
# {'dsn': 'udp://username:password@127.0.0.1:9000/2'}),
# },
# }
#
# from heka.config import client_from_dict_config
# HEKA = client_from_dict_config(HEKA_CONF)
# If you want to allow self-reviews for add-ons/apps, then enable this.
# In production we do not want to allow this.
ALLOW_SELF_REVIEWS = False

Просмотреть файл

@ -14,10 +14,10 @@ Configuration
-------------
Configuration for your unit tests is mostly handled automatically. The only
thing you'll need to ensure is that the database credentials in your
``settings_local.py`` has full permissions to modify a database with ``test-``
prepended to it. For example, if my database name were ``olympia`` this
database would be ``test_olympia``.
thing you'll need to ensure is that the database credentials in your settings
has full permissions to modify a database with ``test_`` prepended to it. By
default the database name is ``olympia``, so the test database is
``test_olympia``.
Running Tests
-------------
@ -43,7 +43,7 @@ the build script at :src:`scripts/build.sh`.
There are a few useful makefile targets that you can use:
Run all the tests using your ``settings_local.py`` file::
Run all the tests::
make test
@ -58,7 +58,7 @@ To fail and stop running tests on the first failure::
If you wish to add arguments, or run a specific test, overload the variables
(check the Makefile for more information)::
make SETTINGS=settings_test ARGS='--verbosity 2 olympia.apps.amo.tests.test_url_prefix:MiddlewareTest.test_get_app' test
make ARGS='--verbosity 2 olympia.apps.amo.tests.test_url_prefix:MiddlewareTest.test_get_app' test
Those targets include some useful options, like the ``--with-id`` which allows
you to re-run only the tests failed from the previous run::

Просмотреть файл

@ -39,7 +39,7 @@ Memcached
We slipped this in with the basic install. The package was
``libmemcached-dev`` on Ubuntu and ``libmemcached`` on OS X. Switch your
``settings_local.py`` to use ::
``local_settings.py`` to use ::
CACHES = {
'default': {
@ -56,7 +56,7 @@ RabbitMQ and Celery
See the :doc:`./celery` page for installation instructions. The
:ref:`example settings <example-settings>` set ``CELERY_ALWAYS_EAGER = True``.
If you're setting up Rabbit and want to use ``celeryd``, make sure you remove
that line from your ``settings_local.py``.
that line from your ``local_settings.py``.
-------------
@ -72,7 +72,7 @@ Redis
On OS X the package is called ``redis``. Get it running with the ``launchctl``
script included in homebrew. To let olympia know about Redis, add this to
``settings_local.py``::
``local_settings.py``::
CACHE_MACHINE_USE_REDIS = True
REDIS_BACKEND = 'redis://'
@ -120,7 +120,7 @@ Learn about Stylus at http://learnboost.github.com/stylus/ ::
cd olympia
npm install
In your ``settings_local.py`` ensure you are pointing to the correct executable
In your ``local_settings.py`` ensure you are pointing to the correct executable
for ``stylus``::
STYLUS_BIN = path('node_modules/stylus/bin/stylus')
@ -135,13 +135,13 @@ LESS at http://lesscss.org.
If you already ran ``npm install`` you don't need to do anything more.
In your ``settings_local.py`` ensure you are pointing to the correct executable
In your ``local_settings.py`` ensure you are pointing to the correct executable
for ``less``::
LESS_BIN = path('node_modules/less/bin/lessc')
You can make the CSS live refresh on save by adding ``#!watch`` to the URL or by
adding the following to your ``settings_local.py``::
adding the following to your ``local_settings.py``::
LESS_LIVE_REFRESH = True

Просмотреть файл

@ -70,15 +70,15 @@ structure of a table, indexing is analagous to storing rows.
For AMO, this will set up all indexes and start the indexing processeses::
./manage.py reindex --settings=your_local_amo_settings
./manage.py reindex
Or you could use the makefile target (using the ``settings_local.py`` file)::
Or you could use the makefile target::
make reindex
If you need to use another settings file and add arguments::
If you need to add arguments::
make SETTINGS=settings_amo ARGS='--with-stats --wipe --force' reindex
make ARGS='--with-stats --wipe --force' reindex
Indexing

Просмотреть файл

@ -139,14 +139,14 @@ add puppet commands like
cp vagrant/manifests/classes/custom-dist.pp vagrant/manifests/classes/custom.pp
For example, if your ``settings_local.py`` file requires additional packages or
For example, if your ``local_settings.py`` file requires additional packages or
Python modules, you'll need to add ``sudo pip install <package>``.
Your ``custom.pp`` file is ignored by git.
Troubleshooting
---------------
If you have already set up Olympia with a custom ``settings_local.py`` file
If you have already set up Olympia with a custom ``local_settings.py`` file
then be sure your database credentials match the defaults::
'NAME': 'olympia',

Просмотреть файл

@ -43,10 +43,6 @@ if you're running a recent version, you can `install them automatically
sudo aptitude install python-dev python-virtualenv libxml2-dev libxslt1-dev libmysqlclient-dev libmemcached-dev libssl-dev swig openssl curl
On versions 12.04 and later, you will need to install a patched version of
M2Crypto instead of the version from PyPI. Please check the `Finish the
install`_ paragraph.
.. _osx-packages:
@ -163,37 +159,6 @@ This runs a command like this::
.. _pip: http://www.pip-installer.org/en/latest/
If you are on a linux box and get a compilation error while installing M2Crypto
like the following::
SWIG/_m2crypto_wrap.c:6116:1: error: unknown type name STACK
... snip a very long output of errors around STACK...
SWIG/_m2crypto_wrap.c:23497:20: error: expected expression before ) token
result = (STACK *)pkcs7_get0_signers(arg1,arg2,arg3);
^
error: command 'gcc' failed with exit status 1
It may be because of a `few reasons`_:
.. _few reasons:
http://blog.rectalogic.com/2013/11/installing-m2crypto-in-python.html
* comment the line starting with ``M2Crypto`` in ``requirements/compiled.txt``
* install the patched package from the Debian repositories (replace
``x86_64-linux-gnu`` by ``i386-linux-gnu`` if you're on a 32bits platform)::
DEB_HOST_MULTIARCH=x86_64-linux-gnu pip install -I --exists-action=w "git+git://anonscm.debian.org/collab-maint/m2crypto.git@debian/0.21.1-3#egg=M2Crypto"
pip install --no-deps -r requirements/dev.txt
* revert your changes to ``requirements/compiled.txt``::
git checkout requirements/compiled.txt
.. _example-settings:
@ -206,19 +171,18 @@ Settings
setup but want to know what has recently changed.
Most of olympia is already configured in ``settings.py``, but there's some
things you need to configure locally. All your local settings go into
``settings_local.py``. The settings template for
developers, included below, is at :src:`docs/settings/settings_local.dev.py`.
things you may want to configure locally. All your local settings go into
``local_settings.py``. The settings template for developers, included below,
is at :src:`docs/settings/local_settings.dev.py`.
.. literalinclude:: /settings/settings_local.dev.py
.. literalinclude:: /settings/local_settings.dev.py
I'm overriding the database parameters from ``settings.py`` and then extending
``INSTALLED_APPS`` and ``MIDDLEWARE_CLASSES`` to include the `Django Debug
Toolbar <http://github.com/robhudson/django-debug-toolbar>`_. It's awesome,
you want it.
I'm extending ``INSTALLED_APPS`` and ``MIDDLEWARE_CLASSES`` to include the
`Django Debug Toolbar <http://github.com/robhudson/django-debug-toolbar>`_.
It's awesome, you want it.
Any file that looks like ``settings_local*`` is for local use only; it will be
ignored by git.
The file ``local_settings.py`` is for local use only; it will be ignored by
git.
Database
@ -229,10 +193,9 @@ our production DB which has been redacted and pruned for development use.
Development snapshots are hosted over at
https://landfill-addons.allizom.org/db/
There is a management command that download and install the landfill
database. You have to create the database first using the following
command filling in the database name from your ``settings_local.py``
(Defaults to ``olympia``)::
There is a management command that download and install the landfill database.
You have to create the database first using the following command filling in
the database name from your settings (Defaults to ``olympia``)::
mysqladmin -uroot create $DB_NAME
@ -280,13 +243,13 @@ More info on schematic: https://github.com/mozilla/schematic
Run the Server
--------------
If you've gotten the system requirements, downloaded ``olympia`` and
``zamboni-lib``, set up your virtualenv with the compiled packages, and
configured your settings and database, you're good to go.
If you've gotten the system requirements, downloaded ``olympia``, set up your
virtualenv with the compiled packages, and configured your settings and
database, you're good to go.
::
./manage.py runserver --settings=settings_local 0.0.0.0:8000
./manage.py runserver
.. note::
@ -294,7 +257,7 @@ configured your settings and database, you're good to go.
http://localhost:8000 in your browser will raise a 500 server error.
If you don't want to run through the :doc:`./advanced-installation`
documentation just right now, you can disable all LESS pre-processing by
adding the following line to your settings_local file::
adding the following line to your ``local_settings.py`` file::
LESS_PREPROCESS = False
@ -338,7 +301,7 @@ Testing
The :ref:`testing` page has more info, but here's the quick way to run
olympia's tests::
./manage.py test --settings=settings_local
./manage.py test
There are a few useful makefile targets that you can use, the simplest one
being::

Просмотреть файл

@ -46,7 +46,7 @@ tree. It sets up sensible defaults, but you can twiddle with these settings:
},
}
If you want to add more to this in ``settings_local.py``, do something like
If you want to add more to this in ``local_settings.py``, do something like
this::
LOGGING['loggers'].update({

1
fabfile.py поставляемый
Просмотреть файл

@ -9,6 +9,7 @@ from fabdeploytools import helpers
import deploysettings as settings
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings_local'
env.key_filename = settings.SSH_KEY
fabdeploytools.envs.loadenv(settings.CLUSTER)

Просмотреть файл

@ -63,8 +63,7 @@ class TestIndexCommand(amo.tests.ESTestCase):
return items
def test_reindexation(self):
if getattr(settings, 'RUNNING_IN_JENKINS', False):
raise SkipTest('Passes locally but fails on Jenkins :(')
raise SkipTest('Fails randomly (bug 1050754)')
# Adding an addon.
addon = amo.tests.addon_factory()

Просмотреть файл

@ -84,7 +84,7 @@ DATABASES = {
'ENGINE': 'django.db.backends.mysql',
'HOST': '',
'PORT': '',
'USER': '',
'USER': 'root',
'PASSWORD': '',
'OPTIONS': {'init_command': 'SET storage_engine=InnoDB'},
'TEST_CHARSET': 'utf8',
@ -97,7 +97,7 @@ DATABASES = {
# database connection, only some values are supported.
SERVICES_DATABASE = {
'NAME': 'olympia',
'USER': '',
'USER': 'root',
'PASSWORD': '',
'HOST': '',
}
@ -1160,7 +1160,7 @@ ENGAGE_ROBOTS = False
READ_ONLY = False
# Turn on read-only mode in settings_local.py by putting this line
# Turn on read-only mode in local_settings.py by putting this line
# at the VERY BOTTOM: read_only_mode(globals())
def read_only_mode(env):
env['READ_ONLY'] = True
@ -1190,7 +1190,7 @@ MAX_PHOTO_UPLOAD_SIZE = MAX_ICON_UPLOAD_SIZE
MAX_PERSONA_UPLOAD_SIZE = 300 * 1024
MAX_REVIEW_ATTACHMENT_UPLOAD_SIZE = 5 * 1024 * 1024
# RECAPTCHA - copy all three statements to settings_local.py
# RECAPTCHA: overload all three statements to local_settings.py with your keys.
RECAPTCHA_PUBLIC_KEY = ''
RECAPTCHA_PRIVATE_KEY = ''
RECAPTCHA_URL = ('https://www.google.com/recaptcha/api/challenge?k=%s' %
@ -1356,7 +1356,7 @@ IN_TEST_SUITE = False
# Until bug 753421 gets fixed, we're skipping ES tests. Sad times. I know.
# Flip this on in your local settings or set an environment variable to
# experience the joy of ES tests.
RUN_ES_TESTS = bool(os.environ.get('RUN_ES_TESTS'))
RUN_ES_TESTS = False
# The configuration for the client that speaks to solitude.
# A tuple of the solitude hosts.

186
manage.py
Просмотреть файл

@ -7,119 +7,81 @@ import warnings
ROOT = os.path.dirname(os.path.abspath(__file__))
if os.path.splitext(os.path.basename(__file__))[0] == 'cProfile':
if os.environ.get('ZAMBONI_PATH'):
ROOT = os.environ['ZAMBONI_PATH']
else:
print 'When using cProfile you must set $ZAMBONI_PATH'
sys.exit(2)
path = lambda *a: os.path.join(ROOT, *a)
prev_sys_path = list(sys.path)
site.addsitedir(path('apps'))
site.addsitedir(path('vendor/lib/python'))
# Move the new items to the front of sys.path. (via virtualenv)
new_sys_path = []
for item in list(sys.path):
if item not in prev_sys_path:
new_sys_path.append(item)
sys.path.remove(item)
sys.path[:0] = new_sys_path
# No third-party imports until we've added all our sitedirs!
from django.core.management import call_command, execute_from_command_line
# Figuring out what settings file to use.
# 1. Look first for the command line setting.
setting = None
if __name__ == '__main__':
for k, v in enumerate(sys.argv):
if v.startswith('--settings'):
setting = v.split('=')[1]
del sys.argv[k]
break
# 2. If not, find the env variable.
if not setting:
setting = os.environ.get('DJANGO_SETTINGS_MODULE', '')
# Django runserver does that double reload of installed settings, settings
# setting to zamboni.settings. We don't want to have zamboni on the path.
if setting.startswith(('zamboni', # typical git clone destination
'workspace', # Jenkins
'project', # vagrant VM
'freddo')):
setting = setting.split('.', 1)[1]
# The average Django user will have DJANGO_SETTINGS_MODULE set to settings
# for our purposes that means, load the default site, so if nothing is
# specified by now, use the default.
if setting in ('settings', ''):
setting = 'settings_local'
# Finally load the settings file that was specified.
from django.utils import importlib
settings = importlib.import_module(setting)
os.environ['DJANGO_SETTINGS_MODULE'] = setting
if not settings.DEBUG:
warnings.simplefilter('ignore')
import session_csrf
session_csrf.monkeypatch()
# Fix jinja's Markup class to not crash when localizers give us bad format
# strings.
from jinja2 import Markup
mod = Markup.__mod__
trans_log = logging.getLogger('z.trans')
#waffle and amo form an import cycle because amo patches waffle and
#waffle loads the user model, so we have to make sure amo gets
#imported before anything else imports waffle.
import amo
# Hardcore monkeypatching action.
import jingo.monkey
jingo.monkey.patch()
def new(self, arg):
try:
return mod(self, arg)
except Exception:
trans_log.error(unicode(self))
return ''
Markup.__mod__ = new
import djcelery
djcelery.setup_loader()
# Import for side-effect: configures our logging handlers.
# pylint: disable-msg=W0611
from lib.log_settings_base import log_configure
log_configure()
import django.conf
newrelic_ini = getattr(django.conf.settings, 'NEWRELIC_INI', None)
load_newrelic = False
if newrelic_ini:
import newrelic.agent
try:
newrelic.agent.initialize(newrelic_ini)
load_newrelic = True
except:
startup_logger = logging.getLogger('z.startup')
startup_logger.exception('Failed to load new relic config.')
if __name__ == "__main__":
path = lambda *a: os.path.join(ROOT, *a)
prev_sys_path = list(sys.path)
site.addsitedir(path('apps'))
site.addsitedir(path('vendor/lib/python'))
# Move the new items to the front of sys.path. (via virtualenv)
new_sys_path = []
for item in list(sys.path):
if item not in prev_sys_path:
new_sys_path.append(item)
sys.path.remove(item)
sys.path[:0] = new_sys_path
# No third-party imports until we've added all our sitedirs!
from django.core.management import call_command, execute_from_command_line
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings")
# Finally load the settings file that was specified.
from django.conf import settings
if not settings.DEBUG:
warnings.simplefilter('ignore')
import session_csrf
session_csrf.monkeypatch()
# Fix jinja's Markup class to not crash when localizers give us bad format
# strings.
from jinja2 import Markup
mod = Markup.__mod__
trans_log = logging.getLogger('z.trans')
# Waffle and amo form an import cycle because amo patches waffle and waffle
# loads the user model, so we have to make sure amo gets imported before
# anything else imports waffle.
import amo # noqa
# Hardcore monkeypatching action.
import jingo.monkey
jingo.monkey.patch()
def new(self, arg):
try:
return mod(self, arg)
except Exception:
trans_log.error(unicode(self))
return ''
Markup.__mod__ = new
import djcelery
djcelery.setup_loader()
# Import for side-effect: configures our logging handlers.
# pylint: disable-msg=W0611
from lib.log_settings_base import log_configure
log_configure()
newrelic_ini = getattr(settings, 'NEWRELIC_INI', None)
load_newrelic = False
if newrelic_ini:
import newrelic.agent
try:
newrelic.agent.initialize(newrelic_ini)
load_newrelic = True
except:
startup_logger = logging.getLogger('z.startup')
startup_logger.exception('Failed to load new relic config.')
# If product details aren't present, get them.
from product_details import product_details
if not product_details.last_update:
@ -127,4 +89,4 @@ if __name__ == "__main__":
call_command('update_product_details')
product_details.__init__() # reload the product details
execute_from_command_line()
execute_from_command_line(sys.argv)

Просмотреть файл

@ -45,14 +45,11 @@ fi
npm install
export PATH="./node_modules/.bin/:${PATH}"
cat > settings_local.py <<SETTINGS
from ${SETTINGS} import *
cat > local_settings <<SETTINGS
CLEANCSS_BIN = 'cleancss'
UGLIFY_BIN = 'uglifyjs'
SETTINGS
export DJANGO_SETTINGS_MODULE=settings_local
# Update the vendor lib.
echo "Updating vendor..."
git submodule --quiet foreach 'git submodule --quiet sync'

Просмотреть файл

@ -12,7 +12,6 @@ VENV=$WORKSPACE/venv
VENDOR=$WORKSPACE/vendor
LOCALE=$WORKSPACE/locale
ES_HOST='jenkins-es20'
SETTINGS=default
echo "Starting build on executor $EXECUTOR_NUMBER..." `date`
@ -22,13 +21,6 @@ if [ -z $1 ]; then
echo "Continuing, but don't say you weren't warned."
fi
if [ -z $2 ]; then
echo "Warning: no settings file specified, using: default"
echo "Usage: ./build.sh <name> <settings>"
else
SETTINGS=$2
fi
echo "Setup..." `date`
# Make sure there's no old pyc files around.
@ -76,11 +68,9 @@ else
RUN_ES_TESTS=True
fi
cat > settings_local.py <<SETTINGS
from lib.settings_base import *
from ${SETTINGS}.settings import *
ROOT_URLCONF = 'lib.urls_base'
LOG_LEVEL = logging.ERROR
cat > local_settings.py <<SETTINGS
from settings_ci import *
DATABASES['default']['NAME'] = 'zamboni_$1'
DATABASES['default']['HOST'] = 'localhost'
DATABASES['default']['USER'] = 'hudson'
@ -90,16 +80,9 @@ DATABASES['default']['TEST_CHARSET'] = 'utf8'
DATABASES['default']['TEST_COLLATION'] = 'utf8_general_ci'
SERVICES_DATABASE['NAME'] = DATABASES['default']['NAME']
SERVICES_DATABASE['USER'] = DATABASES['default']['USER']
CACHES = {
'default': {
'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
'LOCATION': 'localhost:11211',
}
}
CELERY_ALWAYS_EAGER = True
RUN_ES_TESTS = ${RUN_ES_TESTS}
ES_HOSTS = ['${ES_HOST}:9200']
ES_URLS = ['http://%s' % h for h in ES_HOSTS]
RUNNING_IN_JENKINS = True
SETTINGS

Просмотреть файл

@ -1,112 +0,0 @@
# This script should be called from within Jenkins
cd $WORKSPACE
VENV=$WORKSPACE/venv
VENDOR=$WORKSPACE/vendor
LOCALE=$WORKSPACE/locale
ES_HOST='jenkins-es20'
SETTINGS=mkt
echo "Starting build on executor $EXECUTOR_NUMBER..." `date`
if [ -z $1 ]; then
echo "Warning: You should provide a unique name for this job to prevent database collisions."
echo "Usage: ./run_mkt_tests.sh <name> <settings> --with-coverage"
echo "Continuing, but don't say you weren't warned."
fi
if [ -z $2 ]; then
echo "Warning: no settings directory specified, using: ${SETTINGS}"
echo "Usage: ./run_mkt_tests.sh <name> <settings> --with-coverage"
else
SETTINGS=$2
fi
echo "Setup..." `date`
# Make sure there's no old pyc files around.
find . -name '*.pyc' | xargs rm
if [ ! -d "$VENV/bin" ]; then
echo "No virtualenv found. Making one..."
virtualenv $VENV --system-site-packages
fi
source $VENV/bin/activate
pip install -U --exists-action=w --no-deps -q -r requirements/compiled.txt -r requirements/test.txt
# Create paths we want for addons
if [ ! -d "/tmp/warez" ]; then
mkdir /tmp/warez
fi
if [ ! -d "$LOCALE" ]; then
echo "No locale dir? Cloning..."
svn co http://svn.mozilla.org/addons/trunk/site/app/locale/ $LOCALE
fi
if [ ! -d "$VENDOR" ]; then
echo "No vendor lib? Cloning..."
git clone --recursive git://github.com/mozilla/zamboni-lib.git $VENDOR
fi
# Update the vendor lib.
echo "Updating vendor..."
git submodule --quiet foreach 'git submodule --quiet sync'
git submodule --quiet sync && git submodule update --init --recursive
if [ -z $SET_ES_TESTS ]; then
RUN_ES_TESTS=False
else
RUN_ES_TESTS=True
fi
cat > settings_local.py <<SETTINGS
from ${SETTINGS}.settings import *
LOG_LEVEL = logging.ERROR
DATABASES['default']['NAME'] = 'zamboni_$1'
DATABASES['default']['HOST'] = 'localhost'
DATABASES['default']['USER'] = 'hudson'
DATABASES['default']['ENGINE'] = 'mysql_pool'
DATABASES['default']['TEST_NAME'] = 'test_zamboni_$1'
DATABASES['default']['TEST_CHARSET'] = 'utf8'
DATABASES['default']['TEST_COLLATION'] = 'utf8_general_ci'
CACHES = {
'default': {
'BACKEND': 'caching.backends.locmem.LocMemCache',
}
}
CELERY_ALWAYS_EAGER = True
RUN_ES_TESTS = ${RUN_ES_TESTS}
ES_HOSTS = ['${ES_HOST}:9200']
ES_URLS = ['http://%s' % h for h in ES_HOSTS]
ADDONS_PATH = '/tmp/warez'
STATIC_URL = ''
SETTINGS
export DJANGO_SETTINGS_MODULE=settings_local
# Update product details to pull in any changes (namely, 'dbg' locale)
echo "Updating product details..."
python manage.py update_product_details
echo "Starting tests..." `date`
export FORCE_DB='yes sir'
run_tests="python manage.py test -v 2 --noinput --logging-clear-handlers --with-blockage --http-whitelist=127.0.0.1,localhost,${ES_HOST} --with-xunit"
if [[ $3 = '--with-coverage' ]]; then
run_tests+=" --with-coverage --cover-package=mkt --cover-erase --cover-html --cover-xml --cover-xml-file=coverage.xml"
else
if [[ $3 ]]; then
run_tests+=" $3"
fi
fi
exec $run_tests
rv=$?
echo 'shazam!'
exit $rv

Просмотреть файл

@ -12,8 +12,6 @@ import jinja2
from utils import log_configure
import settings_local as settings
# This has to be imported after the settings so statsd knows where to log to.
from django_statsd.clients import statsd

Просмотреть файл

@ -7,9 +7,7 @@ from email.mime.text import MIMEText
from time import time
from urlparse import parse_qsl
from django.utils.http import urlencode
import settings_local as settings
from services.utils import settings
# This has to be imported after the settings so statsd knows where to log to.
from django_statsd.clients import statsd

Просмотреть файл

@ -1,4 +1,3 @@
from datetime import datetime, timedelta
import dictconfig
import logging
import os
@ -6,9 +5,9 @@ import os
# get the right settings module
settingmodule = os.environ.get('DJANGO_SETTINGS_MODULE', 'settings_local')
if settingmodule.startswith(('zamboni', # typical git clone destination
'workspace', # Jenkins
'project', # vagrant VM
'freddo')):
'workspace', # Jenkins
'project', # vagrant VM
'freddo')):
settingmodule = settingmodule.split('.', 1)[1]

Просмотреть файл

@ -1,7 +1,7 @@
import os
import site
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings_local'
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings_local")
wsgidir = os.path.dirname(__file__)
for path in ['../', '../..',

Просмотреть файл

@ -1,7 +1,7 @@
import os
import site
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings_local'
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings_local")
wsgidir = os.path.dirname(__file__)
for path in ['../',

Просмотреть файл

@ -1,7 +1,7 @@
import os
import site
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings_local'
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings_local")
wsgidir = os.path.dirname(__file__)
for path in ['../',

73
settings.py Normal file
Просмотреть файл

@ -0,0 +1,73 @@
"""This is the standard development settings file.
If you need to overload settings, please do so in a local_settings.py file (it
won't be tracked in git).
"""
from lib.settings_base import * # noqa
DEBUG = True
TEMPLATE_DEBUG = DEBUG
DEBUG_PROPAGATE_EXCEPTIONS = DEBUG
# These apps are great during development.
INSTALLED_APPS += (
'django_extensions',
)
# Here we use the LocMemCache backend from cache-machine, as it interprets the
# "0" timeout parameter of ``cache`` in the same way as the Memcached backend:
# as infinity. Django's LocMemCache backend interprets it as a "0 seconds"
# timeout (and thus doesn't cache at all).
CACHES = {
'default': {
'BACKEND': 'caching.backends.locmem.LocMemCache',
'LOCATION': 'olympia',
}
}
HAS_SYSLOG = False # syslog is used if HAS_SYSLOG and NOT DEBUG.
# If you're not running on SSL you'll want this to be False.
SESSION_COOKIE_SECURE = False
SESSION_COOKIE_DOMAIN = None
# Disables custom routing in settings.py so that tasks actually run.
CELERY_ALWAYS_EAGER = True
CELERY_ROUTES = {}
# Disable timeout code during development because it uses the signal module
# which can only run in the main thread. Celery uses threads in dev.
VALIDATOR_TIMEOUT = -1
# If you want to allow self-reviews for add-ons/apps, then enable this.
# In production we do not want to allow this.
ALLOW_SELF_REVIEWS = True
# Assuming you did `npm install` (and not `-g`) like you were supposed to, this
# will be the path to the `stylus` and `lessc` executables.
STYLUS_BIN = path('node_modules/stylus/bin/stylus')
LESS_BIN = path('node_modules/less/bin/lessc')
CLEANCSS_BIN = path('node_modules/clean-css/bin/cleancss')
UGLIFY_BIN = path('node_modules/uglify-js/bin/uglifyjs')
# Locally we typically don't run more than 1 elasticsearch node. So we set
# replicas to zero.
ES_DEFAULT_NUM_REPLICAS = 0
# Overload in local_settings.py to run elasticsearch related tests.
RUN_ES_TESTS = False
SITE_URL = 'http://localhost:8000/'
SERVICES_DOMAIN = 'localhost:8000'
SERVICES_URL = 'http://%s' % SERVICES_DOMAIN
VALIDATE_ADDONS = False
ADDON_COLLECTOR_ID = 1
# If you have settings you want to overload, put them in a local_settings.py.
try:
from local_settings import * # noqa
except ImportError:
pass

10
settings_ci.py Normal file
Просмотреть файл

@ -0,0 +1,10 @@
from settings import * # noqa
LOG_LEVEL = logging.ERROR
CACHES = {
'default': {
'BACKEND': 'caching.backends.memcached.MemcachedCache',
'LOCATION': 'localhost:11211',
}
}

Просмотреть файл

@ -54,12 +54,6 @@ ENABLE_API_ERROR_SERVICE = False
SITE_URL = 'http://testserver'
MOBILE_SITE_URL = ''
CACHES = {
'default': {
'BACKEND': 'caching.backends.locmem.LocMemCache',
}
}
# COUNT() caching can't be invalidated, it just expires after x seconds. This
# is just too annoying for tests, so disable it.
CACHE_COUNT_TIMEOUT = -1

Просмотреть файл

@ -18,7 +18,7 @@ class migrate {
require => [
# Service["mysql"],
Package["python2.6"],
File["$PROJ_DIR/settings_local.py"],
File["$PROJ_DIR/settings.py"],
File["$PROJ_DIR/migrations/264-locale-indexes.py"],
# Exec["fetch_landfill_sql"],
# Exec["load_data"]

Просмотреть файл

@ -10,7 +10,7 @@ class zamboni {
exec { "create_mysql_database":
command => "mysqladmin -uroot create $DB_NAME",
unless => "mysql -uroot -B --skip-column-names -e 'show databases' | /bin/grep '$DB_NAME'",
require => File["$PROJ_DIR/settings_local.py"]
require => File["$PROJ_DIR/settings.py"]
}
exec { "grant_mysql_database":

Просмотреть файл

@ -2,9 +2,9 @@
# the base image again
class zamboni_plus {
file { "$PROJ_DIR/settings_local.py":
file { "$PROJ_DIR/settings.py":
ensure => file,
source => "$PROJ_DIR/docs/settings/settings_local.dev.py",
source => "$PROJ_DIR/settings.py",
replace => false;
}

Просмотреть файл

@ -1,70 +0,0 @@
import os
import site
import sys
from datetime import datetime
# Tell manage that we need to pull in the default settings file.
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings_local_mkt'
# Remember when mod_wsgi loaded this file so we can track it in nagios.
wsgi_loaded = datetime.now()
# Tell celery that we're using Django.
os.environ['CELERY_LOADER'] = 'django'
# Add the zamboni dir to the python path so we can import manage.
wsgidir = os.path.dirname(__file__)
site.addsitedir(os.path.abspath(os.path.join(wsgidir, '../')))
# manage adds /apps, /lib, and /vendor to the Python path.
import manage
import django.conf
import django.core.handlers.wsgi
import django.core.management
import django.utils
# Do validate and activate translations like using `./manage.py runserver`.
# http://blog.dscpl.com.au/2010/03/improved-wsgi-script-for-use-with.html
django.utils.translation.activate(django.conf.settings.LANGUAGE_CODE)
utility = django.core.management.ManagementUtility()
command = utility.fetch_command('runserver')
command.validate()
# This is what mod_wsgi runs.
django_app = django.core.handlers.wsgi.WSGIHandler()
newrelic_ini = getattr(django.conf.settings, 'NEWRELIC_INI', None)
load_newrelic = False
if newrelic_ini:
import newrelic.agent
try:
newrelic.agent.initialize(newrelic_ini)
load_newrelic = True
except:
import logging
startup_logger = logging.getLogger('z.startup')
startup_logger.exception('Failed to load new relic config.')
# Normally we could let WSGIHandler run directly, but while we're dark
# launching, we want to force the script name to be empty so we don't create
# any /z links through reverse. This fixes bug 554576.
def application(env, start_response):
if 'HTTP_X_ZEUS_DL_PT' in env:
env['SCRIPT_URL'] = env['SCRIPT_NAME'] = ''
env['wsgi.loaded'] = wsgi_loaded
env['hostname'] = django.conf.settings.HOSTNAME
env['datetime'] = str(datetime.now())
return django_app(env, start_response)
if load_newrelic:
application = newrelic.agent.wsgi_application()(application)
# Uncomment this to figure out what's going on with the mod_wsgi environment.
# def application(env, start_response):
# start_response('200 OK', [('Content-Type', 'text/plain')])
# return '\n'.join('%r: %r' % item for item in sorted(env.items()))
# vim: ft=python

Просмотреть файл

@ -4,7 +4,7 @@ import sys
from datetime import datetime
# Tell manage that we need to pull in the default settings file.
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings_local'
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
# Remember when mod_wsgi loaded this file so we can track it in nagios.
wsgi_loaded = datetime.now()