diff --git a/.gitignore b/.gitignore index c50a47848c..ad40d37ce8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .env settings_local.py* settings_local_*.py +local_settings.py shellng_local.py *.py[co] *.sw[po] diff --git a/.travis.yml b/.travis.yml index 4b94a6887d..725126e0c2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,7 @@ language: python python: - "2.7" -service: - - redis-server +services: - memcached before_install: - wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-0.90.13.deb && sudo dpkg -i elasticsearch-0.90.13.deb @@ -12,14 +11,12 @@ install: - pip install --no-deps -r requirements/test.txt --find-links https://pyrepo.addons.mozilla.org/ - pip install --no-deps -r requirements/compiled.txt --find-links https://pyrepo.addons.mozilla.org/ - npm install - - export PATH="./node_modules/.bin/:${PATH}" before_script: - mysql -e 'create database olympia;' - - echo -e "from lib.settings_base import *\nCELERY_ALWAYS_EAGER = True\nRUNNING_IN_JENKINS = True" > settings_local.py - make update_submodules - make update_assets script: - - make test + - DJANGO_SETTINGS_MODULE=settings_ci make test notifications: irc: channels: diff --git a/Makefile b/Makefile index ddadb6ace2..31ad9f36fe 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,3 @@ -# You can set these variables from the command line. -DJANGO=python manage.py -SETTINGS=settings_local -RUN_ES_TESTS=1 - .PHONY: help docs test test_force_db tdd test_failed update_code update_submodules update_deps update_db update_assets update_landfill full_update reindex help: @@ -20,23 +15,21 @@ help: @echo " update_landfill to load the landfill database data" @echo " reindex to reindex everything in elasticsearch, for AMO" @echo "Check the Makefile to know exactly what each target is doing. If you see a " - @echo "target using something like $(SETTINGS), you can make it use another value:" - @echo " make SETTINGS=settings_mine docs" docs: $(MAKE) -C docs html test: - RUN_ES_TESTS=$(RUN_ES_TESTS) $(DJANGO) test --settings=$(SETTINGS) --with-blockage --noinput --logging-clear-handlers --with-id -v 2 $(ARGS) + python manage.py test --with-blockage --noinput --logging-clear-handlers --with-id -v 2 $(ARGS) test_force_db: - RUN_ES_TESTS=$(RUN_ES_TESTS) FORCE_DB=1 $(DJANGO) test --settings=$(SETTINGS) --with-blockage --noinput --logging-clear-handlers --with-id -v 2 $(ARGS) + FORCE_DB=1 python manage.py test --with-blockage --noinput --logging-clear-handlers --with-id -v 2 $(ARGS) tdd: - RUN_ES_TESTS=$(RUN_ES_TESTS) $(DJANGO) test --settings=$(SETTINGS) --with-blockage --noinput --failfast --pdb --with-id -v 2 $(ARGS) + python manage.py test --with-blockage --noinput --failfast --pdb --with-id -v 2 $(ARGS) test_failed: - RUN_ES_TESTS=$(RUN_ES_TESTS) $(DJANGO) test --settings=$(SETTINGS) --with-blockage --noinput --logging-clear-handlers --with-id -v 2 --failed $(ARGS) + python manage.py test --with-blockage --noinput --logging-clear-handlers --with-id -v 2 --failed $(ARGS) update_code: git checkout master && git pull @@ -54,13 +47,13 @@ update_db: schematic migrations update_assets: - $(DJANGO) compress_assets --settings=$(SETTINGS) - $(DJANGO) collectstatic --settings=$(SETTINGS) --noinput + python manage.py compress_assets + python manage.py collectstatic --noinput full_update: update_code update_deps update_db update_assets update_landfill: - $(DJANGO) install_landfill --settings=$(SETTINGS) $(ARGS) + python manage.py install_landfill $(ARGS) reindex: - $(DJANGO) reindex --settings=$(SETTINGS) $(ARGS) + python manage.py reindex $(ARGS) diff --git a/apps/addons/tests/test_theme_update.py b/apps/addons/tests/test_theme_update.py index 67829696ce..c242cd0a11 100644 --- a/apps/addons/tests/test_theme_update.py +++ b/apps/addons/tests/test_theme_update.py @@ -144,9 +144,9 @@ class TestThemeUpdate(amo.tests.TestCase): up.get_update() image_path = up.image_path('foo.png') # This is ugly. It's needed because services.theme_update imports - # settings_local, and settings_test is overriding MEDIA_ROOT. - import settings_local - with self.settings(MEDIA_ROOT=settings_local.MEDIA_ROOT): + # settings, and settings_test is overriding MEDIA_ROOT. + import settings + with self.settings(MEDIA_ROOT=settings.MEDIA_ROOT): assert user_media_path('addons') in image_path def test_image_url(self): diff --git a/apps/addons/tests/test_update.py b/apps/addons/tests/test_update.py index c52c73b843..77620975e2 100644 --- a/apps/addons/tests/test_update.py +++ b/apps/addons/tests/test_update.py @@ -14,7 +14,6 @@ from amo.helpers import user_media_url from applications.models import Application, AppVersion from files.models import File from services import update -import settings_local from versions.models import ApplicationsVersions, Version @@ -589,14 +588,6 @@ class TestResponse(VersionCheckMixin, amo.tests.TestCase): self.mac = amo.PLATFORM_MAC self.win = amo.PLATFORM_WIN - self.old_debug = settings_local.DEBUG - - settings_local.DEBUG = False - - def tearDown(self): - settings_local.DEBUG = self.old_debug - super(TestResponse, self).tearDown() - def test_bad_guid(self): data = self.good_data.copy() data["id"] = "garbage" diff --git a/apps/search/tests/test_views.py b/apps/search/tests/test_views.py index 39a9152480..58fe97d211 100644 --- a/apps/search/tests/test_views.py +++ b/apps/search/tests/test_views.py @@ -906,8 +906,7 @@ class TestCollectionSearch(SearchBase): eq_(doc('.listing-footer').length, 0) def test_results_name_query(self): - if getattr(settings, 'RUNNING_IN_JENKINS', False): - raise SkipTest('Passes locally but fails on Jenkins :(') + raise SkipTest('Fails randomly (bug 1050754)') self._generate() diff --git a/apps/zadmin/views.py b/apps/zadmin/views.py index 3307dd7424..e18abc0ed5 100644 --- a/apps/zadmin/views.py +++ b/apps/zadmin/views.py @@ -165,7 +165,7 @@ def show_settings(request): for i in settings.HERA: settings_dict['HERA'].append(debug.cleanse_setting('HERA', i)) - # Retain this so that legacy PAYPAL_CGI_AUTH variables in settings_local + # Retain this so that legacy PAYPAL_CGI_AUTH variables in local settings # are not exposed. for i in ['PAYPAL_EMBEDDED_AUTH', 'PAYPAL_CGI_AUTH', 'GOOGLE_ANALYTICS_CREDENTIALS']: diff --git a/docs/settings/local_settings.dev.py b/docs/settings/local_settings.dev.py new file mode 100644 index 0000000000..51b8651a7f --- /dev/null +++ b/docs/settings/local_settings.dev.py @@ -0,0 +1,19 @@ +from settings import * # noqa + +# To activate the Django debug toolbar. +INSTALLED_APPS += ( + 'debug_toolbar', + 'fixture_magic', +) +DEBUG_TOOLBAR_PATCH_SETTINGS = False # Prevent DDT from patching the settings. + +INTERNAL_IPS = ('127.0.0.1',) +MIDDLEWARE_CLASSES += ('debug_toolbar.middleware.DebugToolbarMiddleware',) + +# Use the real stuff. +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', + 'LOCATION': 'localhost:11211', + } +} diff --git a/docs/settings/settings_local.dev.py b/docs/settings/settings_local.dev.py deleted file mode 100644 index 291dce6130..0000000000 --- a/docs/settings/settings_local.dev.py +++ /dev/null @@ -1,94 +0,0 @@ -from default.settings import * - -DEBUG = True -TEMPLATE_DEBUG = DEBUG -DEBUG_PROPAGATE_EXCEPTIONS = DEBUG - -# These apps are great during development. -INSTALLED_APPS += ( - 'debug_toolbar', - 'django.contrib.staticfiles', # Needed by the latest versions of DDT. - 'django_extensions', - 'fixture_magic', -) -# Prevent DDT from patching the settings automatically: this would result in -# circular imports for us. See DDT docs for more information. -DEBUG_TOOLBAR_PATCH_SETTINGS = False - -# You want one of the caching backends. Dummy won't do any caching, locmem is -# cleared every time you restart the server, and memcached is what we run in -# production. -#CACHES = { -# 'default': { -# 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', -# 'LOCATION': 'localhost:11211', -# } -#} -# Here we use the LocMemCache backend from cache-machine, as it interprets the -# "0" timeout parameter of ``cache`` in the same way as the Memcached backend: -# as infinity. Django's LocMemCache backend interprets it as a "0 seconds" -# timeout (and thus doesn't cache at all). -CACHES = { - 'default': { - 'BACKEND': 'caching.backends.locmem.LocMemCache', - 'LOCATION': 'olympia', - } -} -# Caching is required for CSRF to work, please do not use the dummy cache. - -DATABASES = { - 'default': { - 'NAME': 'olympia', - 'ENGINE': 'django.db.backends.mysql', - 'USER': 'root', - 'PASSWORD': '', - 'OPTIONS': {'init_command': 'SET storage_engine=InnoDB'}, - 'TEST_CHARSET': 'utf8', - 'TEST_COLLATION': 'utf8_general_ci', - }, -} - -# Skip indexing ES to speed things up? -SKIP_SEARCH_INDEX = False - -LOG_LEVEL = logging.DEBUG -HAS_SYSLOG = False - -# For debug toolbar. -if DEBUG: - INTERNAL_IPS = ('127.0.0.1',) - MIDDLEWARE_CLASSES += ('debug_toolbar.middleware.DebugToolbarMiddleware',) - -# If you're not running on SSL you'll want this to be False. -SESSION_COOKIE_SECURE = False -SESSION_COOKIE_DOMAIN = None - -# Run tasks immediately, don't try using the queue. -CELERY_ALWAYS_EAGER = True - -# Disables custom routing in settings.py so that tasks actually run. -CELERY_ROUTES = {} - -# Disable timeout code during development because it uses the signal module -# which can only run in the main thread. Celery uses threads in dev. -VALIDATOR_TIMEOUT = -1 - -# The user id to use when logging in tasks. You should set this to a user that -# exists in your site. -# TASK_USER_ID = 1 - -# If you want to allow self-reviews for add-ons/apps, then enable this. -# In production we do not want to allow this. -ALLOW_SELF_REVIEWS = True - -# If you want to skip pre-generation locally, disable it: -PRE_GENERATE_APKS = False - -# Assuming you did `npm install` (and not `-g`) like you were supposed to, -# this will be the path to the `stylus` and `lessc` executables. -STYLUS_BIN = path('node_modules/stylus/bin/stylus') -LESS_BIN = path('node_modules/less/bin/lessc') - -# Locally we typically don't run more than 1 elasticsearch node. So we set -# replicas to zero. -ES_DEFAULT_NUM_REPLICAS = 0 diff --git a/docs/settings/settings_local.prod.py b/docs/settings/settings_local.prod.py deleted file mode 100644 index a291bb8fa6..0000000000 --- a/docs/settings/settings_local.prod.py +++ /dev/null @@ -1,70 +0,0 @@ -from settings import * - -DEBUG = False -TEMPLATE_DEBUG = False - -# The default database should point to the master. -DATABASES = { - 'default': { - 'NAME': 'olympia', - 'ENGINE': 'django.db.backends.mysql', - 'HOST': '', - 'PORT': '', - 'USER': '', - 'PASSWORD': '', - 'OPTIONS': {'init_command': 'SET storage_engine=InnoDB'}, - }, - 'slave': { - 'NAME': 'olympia', - 'ENGINE': 'django.db.backends.mysql', - 'HOST': '', - 'PORT': '', - 'USER': '', - 'PASSWORD': '', - 'OPTIONS': {'init_command': 'SET storage_engine=InnoDB'}, - }, -} - -# Put the aliases for slave databases in this list. -SLAVE_DATABASES = ['slave'] - -# Use IP:PORT pairs separated by semicolons. -CACHES = { - 'default': { - 'BACKEND': 'caching.backends.memcached.MemcachedCache', - 'LOCATION': ['localhost:11211', 'localhost:11212'], - 'TIMEOUT': 500, - } -} - -# This is used to hash some things in Django. -SECRET_KEY = 'replace me with something long' - -LOG_LEVEL = logging.WARNING - -DEBUG_PROPAGATE_EXCEPTIONS = DEBUG - - -# Sample heka configuration. Uncommented, this would override what is in -# lib/settings_base.py. - -# HEKA_CONF = { -# 'logger': 'olympia', -# 'stream': { -# 'class': 'heka.streams.UdpStream', -# 'host': ['10.0.1.5', '10.0.1.10'] -# 'port': 5566 -# }, -# 'plugins': { -# 'raven': ('heka_raven.raven_plugin.config_plugin', -# {'dsn': 'udp://username:password@127.0.0.1:9000/2'}), -# }, -# } -# -# from heka.config import client_from_dict_config -# HEKA = client_from_dict_config(HEKA_CONF) - - -# If you want to allow self-reviews for add-ons/apps, then enable this. -# In production we do not want to allow this. -ALLOW_SELF_REVIEWS = False diff --git a/docs/topics/hacking/testing.rst b/docs/topics/hacking/testing.rst index 16221e7cc2..c799daa4af 100644 --- a/docs/topics/hacking/testing.rst +++ b/docs/topics/hacking/testing.rst @@ -14,10 +14,10 @@ Configuration ------------- Configuration for your unit tests is mostly handled automatically. The only -thing you'll need to ensure is that the database credentials in your -``settings_local.py`` has full permissions to modify a database with ``test-`` -prepended to it. For example, if my database name were ``olympia`` this -database would be ``test_olympia``. +thing you'll need to ensure is that the database credentials in your settings +has full permissions to modify a database with ``test_`` prepended to it. By +default the database name is ``olympia``, so the test database is +``test_olympia``. Running Tests ------------- @@ -43,7 +43,7 @@ the build script at :src:`scripts/build.sh`. There are a few useful makefile targets that you can use: -Run all the tests using your ``settings_local.py`` file:: +Run all the tests:: make test @@ -58,7 +58,7 @@ To fail and stop running tests on the first failure:: If you wish to add arguments, or run a specific test, overload the variables (check the Makefile for more information):: - make SETTINGS=settings_test ARGS='--verbosity 2 olympia.apps.amo.tests.test_url_prefix:MiddlewareTest.test_get_app' test + make ARGS='--verbosity 2 olympia.apps.amo.tests.test_url_prefix:MiddlewareTest.test_get_app' test Those targets include some useful options, like the ``--with-id`` which allows you to re-run only the tests failed from the previous run:: diff --git a/docs/topics/install-olympia/advanced-installation.rst b/docs/topics/install-olympia/advanced-installation.rst index 74cc0ebbfc..55c3aea339 100644 --- a/docs/topics/install-olympia/advanced-installation.rst +++ b/docs/topics/install-olympia/advanced-installation.rst @@ -39,7 +39,7 @@ Memcached We slipped this in with the basic install. The package was ``libmemcached-dev`` on Ubuntu and ``libmemcached`` on OS X. Switch your -``settings_local.py`` to use :: +``local_settings.py`` to use :: CACHES = { 'default': { @@ -56,7 +56,7 @@ RabbitMQ and Celery See the :doc:`./celery` page for installation instructions. The :ref:`example settings ` set ``CELERY_ALWAYS_EAGER = True``. If you're setting up Rabbit and want to use ``celeryd``, make sure you remove -that line from your ``settings_local.py``. +that line from your ``local_settings.py``. ------------- @@ -72,7 +72,7 @@ Redis On OS X the package is called ``redis``. Get it running with the ``launchctl`` script included in homebrew. To let olympia know about Redis, add this to -``settings_local.py``:: +``local_settings.py``:: CACHE_MACHINE_USE_REDIS = True REDIS_BACKEND = 'redis://' @@ -120,7 +120,7 @@ Learn about Stylus at http://learnboost.github.com/stylus/ :: cd olympia npm install -In your ``settings_local.py`` ensure you are pointing to the correct executable +In your ``local_settings.py`` ensure you are pointing to the correct executable for ``stylus``:: STYLUS_BIN = path('node_modules/stylus/bin/stylus') @@ -135,13 +135,13 @@ LESS at http://lesscss.org. If you already ran ``npm install`` you don't need to do anything more. -In your ``settings_local.py`` ensure you are pointing to the correct executable +In your ``local_settings.py`` ensure you are pointing to the correct executable for ``less``:: LESS_BIN = path('node_modules/less/bin/lessc') You can make the CSS live refresh on save by adding ``#!watch`` to the URL or by -adding the following to your ``settings_local.py``:: +adding the following to your ``local_settings.py``:: LESS_LIVE_REFRESH = True diff --git a/docs/topics/install-olympia/elasticsearch.rst b/docs/topics/install-olympia/elasticsearch.rst index d1f2077780..d554d76eea 100644 --- a/docs/topics/install-olympia/elasticsearch.rst +++ b/docs/topics/install-olympia/elasticsearch.rst @@ -70,15 +70,15 @@ structure of a table, indexing is analagous to storing rows. For AMO, this will set up all indexes and start the indexing processeses:: - ./manage.py reindex --settings=your_local_amo_settings + ./manage.py reindex -Or you could use the makefile target (using the ``settings_local.py`` file):: +Or you could use the makefile target:: make reindex -If you need to use another settings file and add arguments:: +If you need to add arguments:: - make SETTINGS=settings_amo ARGS='--with-stats --wipe --force' reindex + make ARGS='--with-stats --wipe --force' reindex Indexing diff --git a/docs/topics/install-olympia/install-with-vagrant.rst b/docs/topics/install-olympia/install-with-vagrant.rst index d78aa9dc9e..f58e42d075 100644 --- a/docs/topics/install-olympia/install-with-vagrant.rst +++ b/docs/topics/install-olympia/install-with-vagrant.rst @@ -139,14 +139,14 @@ add puppet commands like cp vagrant/manifests/classes/custom-dist.pp vagrant/manifests/classes/custom.pp -For example, if your ``settings_local.py`` file requires additional packages or +For example, if your ``local_settings.py`` file requires additional packages or Python modules, you'll need to add ``sudo pip install ``. Your ``custom.pp`` file is ignored by git. Troubleshooting --------------- -If you have already set up Olympia with a custom ``settings_local.py`` file +If you have already set up Olympia with a custom ``local_settings.py`` file then be sure your database credentials match the defaults:: 'NAME': 'olympia', diff --git a/docs/topics/install-olympia/installation.rst b/docs/topics/install-olympia/installation.rst index d6ccd42282..cb82c3beb7 100644 --- a/docs/topics/install-olympia/installation.rst +++ b/docs/topics/install-olympia/installation.rst @@ -43,10 +43,6 @@ if you're running a recent version, you can `install them automatically sudo aptitude install python-dev python-virtualenv libxml2-dev libxslt1-dev libmysqlclient-dev libmemcached-dev libssl-dev swig openssl curl -On versions 12.04 and later, you will need to install a patched version of -M2Crypto instead of the version from PyPI. Please check the `Finish the -install`_ paragraph. - .. _osx-packages: @@ -163,37 +159,6 @@ This runs a command like this:: .. _pip: http://www.pip-installer.org/en/latest/ -If you are on a linux box and get a compilation error while installing M2Crypto -like the following:: - - SWIG/_m2crypto_wrap.c:6116:1: error: unknown type name ‘STACK’ - - ... snip a very long output of errors around STACK... - - SWIG/_m2crypto_wrap.c:23497:20: error: expected expression before ‘)’ token - - result = (STACK *)pkcs7_get0_signers(arg1,arg2,arg3); - - ^ - - error: command 'gcc' failed with exit status 1 - -It may be because of a `few reasons`_: - -.. _few reasons: - http://blog.rectalogic.com/2013/11/installing-m2crypto-in-python.html - -* comment the line starting with ``M2Crypto`` in ``requirements/compiled.txt`` -* install the patched package from the Debian repositories (replace - ``x86_64-linux-gnu`` by ``i386-linux-gnu`` if you're on a 32bits platform):: - - DEB_HOST_MULTIARCH=x86_64-linux-gnu pip install -I --exists-action=w "git+git://anonscm.debian.org/collab-maint/m2crypto.git@debian/0.21.1-3#egg=M2Crypto" - pip install --no-deps -r requirements/dev.txt - -* revert your changes to ``requirements/compiled.txt``:: - - git checkout requirements/compiled.txt - .. _example-settings: @@ -206,19 +171,18 @@ Settings setup but want to know what has recently changed. Most of olympia is already configured in ``settings.py``, but there's some -things you need to configure locally. All your local settings go into -``settings_local.py``. The settings template for -developers, included below, is at :src:`docs/settings/settings_local.dev.py`. +things you may want to configure locally. All your local settings go into +``local_settings.py``. The settings template for developers, included below, +is at :src:`docs/settings/local_settings.dev.py`. -.. literalinclude:: /settings/settings_local.dev.py +.. literalinclude:: /settings/local_settings.dev.py -I'm overriding the database parameters from ``settings.py`` and then extending -``INSTALLED_APPS`` and ``MIDDLEWARE_CLASSES`` to include the `Django Debug -Toolbar `_. It's awesome, -you want it. +I'm extending ``INSTALLED_APPS`` and ``MIDDLEWARE_CLASSES`` to include the +`Django Debug Toolbar `_. +It's awesome, you want it. -Any file that looks like ``settings_local*`` is for local use only; it will be -ignored by git. +The file ``local_settings.py`` is for local use only; it will be ignored by +git. Database @@ -229,10 +193,9 @@ our production DB which has been redacted and pruned for development use. Development snapshots are hosted over at https://landfill-addons.allizom.org/db/ -There is a management command that download and install the landfill -database. You have to create the database first using the following -command filling in the database name from your ``settings_local.py`` -(Defaults to ``olympia``):: +There is a management command that download and install the landfill database. +You have to create the database first using the following command filling in +the database name from your settings (Defaults to ``olympia``):: mysqladmin -uroot create $DB_NAME @@ -280,13 +243,13 @@ More info on schematic: https://github.com/mozilla/schematic Run the Server -------------- -If you've gotten the system requirements, downloaded ``olympia`` and -``zamboni-lib``, set up your virtualenv with the compiled packages, and -configured your settings and database, you're good to go. +If you've gotten the system requirements, downloaded ``olympia``, set up your +virtualenv with the compiled packages, and configured your settings and +database, you're good to go. :: - ./manage.py runserver --settings=settings_local 0.0.0.0:8000 + ./manage.py runserver .. note:: @@ -294,7 +257,7 @@ configured your settings and database, you're good to go. http://localhost:8000 in your browser will raise a 500 server error. If you don't want to run through the :doc:`./advanced-installation` documentation just right now, you can disable all LESS pre-processing by - adding the following line to your settings_local file:: + adding the following line to your ``local_settings.py`` file:: LESS_PREPROCESS = False @@ -338,7 +301,7 @@ Testing The :ref:`testing` page has more info, but here's the quick way to run olympia's tests:: - ./manage.py test --settings=settings_local + ./manage.py test There are a few useful makefile targets that you can use, the simplest one being:: diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 2a392ec380..bf7bb8c54b 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -46,7 +46,7 @@ tree. It sets up sensible defaults, but you can twiddle with these settings: }, } - If you want to add more to this in ``settings_local.py``, do something like + If you want to add more to this in ``local_settings.py``, do something like this:: LOGGING['loggers'].update({ diff --git a/fabfile.py b/fabfile.py index 4118044783..6add245be0 100644 --- a/fabfile.py +++ b/fabfile.py @@ -9,6 +9,7 @@ from fabdeploytools import helpers import deploysettings as settings +os.environ['DJANGO_SETTINGS_MODULE'] = 'settings_local' env.key_filename = settings.SSH_KEY fabdeploytools.envs.loadenv(settings.CLUSTER) diff --git a/lib/es/tests/test_commands.py b/lib/es/tests/test_commands.py index 93e0ae158b..17573e983b 100644 --- a/lib/es/tests/test_commands.py +++ b/lib/es/tests/test_commands.py @@ -63,8 +63,7 @@ class TestIndexCommand(amo.tests.ESTestCase): return items def test_reindexation(self): - if getattr(settings, 'RUNNING_IN_JENKINS', False): - raise SkipTest('Passes locally but fails on Jenkins :(') + raise SkipTest('Fails randomly (bug 1050754)') # Adding an addon. addon = amo.tests.addon_factory() diff --git a/lib/settings_base.py b/lib/settings_base.py index 1179f7488f..e76076204f 100644 --- a/lib/settings_base.py +++ b/lib/settings_base.py @@ -84,7 +84,7 @@ DATABASES = { 'ENGINE': 'django.db.backends.mysql', 'HOST': '', 'PORT': '', - 'USER': '', + 'USER': 'root', 'PASSWORD': '', 'OPTIONS': {'init_command': 'SET storage_engine=InnoDB'}, 'TEST_CHARSET': 'utf8', @@ -97,7 +97,7 @@ DATABASES = { # database connection, only some values are supported. SERVICES_DATABASE = { 'NAME': 'olympia', - 'USER': '', + 'USER': 'root', 'PASSWORD': '', 'HOST': '', } @@ -1160,7 +1160,7 @@ ENGAGE_ROBOTS = False READ_ONLY = False -# Turn on read-only mode in settings_local.py by putting this line +# Turn on read-only mode in local_settings.py by putting this line # at the VERY BOTTOM: read_only_mode(globals()) def read_only_mode(env): env['READ_ONLY'] = True @@ -1190,7 +1190,7 @@ MAX_PHOTO_UPLOAD_SIZE = MAX_ICON_UPLOAD_SIZE MAX_PERSONA_UPLOAD_SIZE = 300 * 1024 MAX_REVIEW_ATTACHMENT_UPLOAD_SIZE = 5 * 1024 * 1024 -# RECAPTCHA - copy all three statements to settings_local.py +# RECAPTCHA: overload all three statements to local_settings.py with your keys. RECAPTCHA_PUBLIC_KEY = '' RECAPTCHA_PRIVATE_KEY = '' RECAPTCHA_URL = ('https://www.google.com/recaptcha/api/challenge?k=%s' % @@ -1356,7 +1356,7 @@ IN_TEST_SUITE = False # Until bug 753421 gets fixed, we're skipping ES tests. Sad times. I know. # Flip this on in your local settings or set an environment variable to # experience the joy of ES tests. -RUN_ES_TESTS = bool(os.environ.get('RUN_ES_TESTS')) +RUN_ES_TESTS = False # The configuration for the client that speaks to solitude. # A tuple of the solitude hosts. diff --git a/manage.py b/manage.py index e5a21f78a6..ac5fe96780 100755 --- a/manage.py +++ b/manage.py @@ -7,119 +7,81 @@ import warnings ROOT = os.path.dirname(os.path.abspath(__file__)) -if os.path.splitext(os.path.basename(__file__))[0] == 'cProfile': - if os.environ.get('ZAMBONI_PATH'): - ROOT = os.environ['ZAMBONI_PATH'] - else: - print 'When using cProfile you must set $ZAMBONI_PATH' - sys.exit(2) - -path = lambda *a: os.path.join(ROOT, *a) - -prev_sys_path = list(sys.path) - -site.addsitedir(path('apps')) -site.addsitedir(path('vendor/lib/python')) - -# Move the new items to the front of sys.path. (via virtualenv) -new_sys_path = [] -for item in list(sys.path): - if item not in prev_sys_path: - new_sys_path.append(item) - sys.path.remove(item) -sys.path[:0] = new_sys_path - - -# No third-party imports until we've added all our sitedirs! -from django.core.management import call_command, execute_from_command_line - -# Figuring out what settings file to use. -# 1. Look first for the command line setting. -setting = None -if __name__ == '__main__': - for k, v in enumerate(sys.argv): - if v.startswith('--settings'): - setting = v.split('=')[1] - del sys.argv[k] - break - -# 2. If not, find the env variable. -if not setting: - setting = os.environ.get('DJANGO_SETTINGS_MODULE', '') - -# Django runserver does that double reload of installed settings, settings -# setting to zamboni.settings. We don't want to have zamboni on the path. -if setting.startswith(('zamboni', # typical git clone destination - 'workspace', # Jenkins - 'project', # vagrant VM - 'freddo')): - setting = setting.split('.', 1)[1] - -# The average Django user will have DJANGO_SETTINGS_MODULE set to settings -# for our purposes that means, load the default site, so if nothing is -# specified by now, use the default. -if setting in ('settings', ''): - setting = 'settings_local' - -# Finally load the settings file that was specified. -from django.utils import importlib -settings = importlib.import_module(setting) -os.environ['DJANGO_SETTINGS_MODULE'] = setting - -if not settings.DEBUG: - warnings.simplefilter('ignore') - -import session_csrf -session_csrf.monkeypatch() - -# Fix jinja's Markup class to not crash when localizers give us bad format -# strings. -from jinja2 import Markup -mod = Markup.__mod__ -trans_log = logging.getLogger('z.trans') - - -#waffle and amo form an import cycle because amo patches waffle and -#waffle loads the user model, so we have to make sure amo gets -#imported before anything else imports waffle. -import amo - -# Hardcore monkeypatching action. -import jingo.monkey -jingo.monkey.patch() - -def new(self, arg): - try: - return mod(self, arg) - except Exception: - trans_log.error(unicode(self)) - return '' - -Markup.__mod__ = new - -import djcelery -djcelery.setup_loader() - -# Import for side-effect: configures our logging handlers. -# pylint: disable-msg=W0611 -from lib.log_settings_base import log_configure -log_configure() - -import django.conf -newrelic_ini = getattr(django.conf.settings, 'NEWRELIC_INI', None) -load_newrelic = False - -if newrelic_ini: - import newrelic.agent - try: - newrelic.agent.initialize(newrelic_ini) - load_newrelic = True - except: - startup_logger = logging.getLogger('z.startup') - startup_logger.exception('Failed to load new relic config.') - if __name__ == "__main__": + path = lambda *a: os.path.join(ROOT, *a) + + prev_sys_path = list(sys.path) + + site.addsitedir(path('apps')) + site.addsitedir(path('vendor/lib/python')) + + # Move the new items to the front of sys.path. (via virtualenv) + new_sys_path = [] + for item in list(sys.path): + if item not in prev_sys_path: + new_sys_path.append(item) + sys.path.remove(item) + sys.path[:0] = new_sys_path + + # No third-party imports until we've added all our sitedirs! + from django.core.management import call_command, execute_from_command_line + + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings") + + # Finally load the settings file that was specified. + from django.conf import settings + + if not settings.DEBUG: + warnings.simplefilter('ignore') + + import session_csrf + session_csrf.monkeypatch() + + # Fix jinja's Markup class to not crash when localizers give us bad format + # strings. + from jinja2 import Markup + mod = Markup.__mod__ + trans_log = logging.getLogger('z.trans') + + # Waffle and amo form an import cycle because amo patches waffle and waffle + # loads the user model, so we have to make sure amo gets imported before + # anything else imports waffle. + import amo # noqa + + # Hardcore monkeypatching action. + import jingo.monkey + jingo.monkey.patch() + + def new(self, arg): + try: + return mod(self, arg) + except Exception: + trans_log.error(unicode(self)) + return '' + + Markup.__mod__ = new + + import djcelery + djcelery.setup_loader() + + # Import for side-effect: configures our logging handlers. + # pylint: disable-msg=W0611 + from lib.log_settings_base import log_configure + log_configure() + + newrelic_ini = getattr(settings, 'NEWRELIC_INI', None) + load_newrelic = False + + if newrelic_ini: + import newrelic.agent + try: + newrelic.agent.initialize(newrelic_ini) + load_newrelic = True + except: + startup_logger = logging.getLogger('z.startup') + startup_logger.exception('Failed to load new relic config.') + # If product details aren't present, get them. from product_details import product_details if not product_details.last_update: @@ -127,4 +89,4 @@ if __name__ == "__main__": call_command('update_product_details') product_details.__init__() # reload the product details - execute_from_command_line() + execute_from_command_line(sys.argv) diff --git a/scripts/build-assets.sh b/scripts/build-assets.sh index df5ac5ac35..70c2e402b9 100755 --- a/scripts/build-assets.sh +++ b/scripts/build-assets.sh @@ -45,14 +45,11 @@ fi npm install export PATH="./node_modules/.bin/:${PATH}" -cat > settings_local.py < local_settings < " -else - SETTINGS=$2 -fi - echo "Setup..." `date` # Make sure there's no old pyc files around. @@ -76,11 +68,9 @@ else RUN_ES_TESTS=True fi -cat > settings_local.py < local_settings.py < --with-coverage" - echo "Continuing, but don't say you weren't warned." -fi - -if [ -z $2 ]; then - echo "Warning: no settings directory specified, using: ${SETTINGS}" - echo "Usage: ./run_mkt_tests.sh --with-coverage" -else - SETTINGS=$2 -fi - -echo "Setup..." `date` - -# Make sure there's no old pyc files around. -find . -name '*.pyc' | xargs rm - -if [ ! -d "$VENV/bin" ]; then - echo "No virtualenv found. Making one..." - virtualenv $VENV --system-site-packages -fi - -source $VENV/bin/activate - -pip install -U --exists-action=w --no-deps -q -r requirements/compiled.txt -r requirements/test.txt - -# Create paths we want for addons -if [ ! -d "/tmp/warez" ]; then - mkdir /tmp/warez -fi - -if [ ! -d "$LOCALE" ]; then - echo "No locale dir? Cloning..." - svn co http://svn.mozilla.org/addons/trunk/site/app/locale/ $LOCALE -fi - -if [ ! -d "$VENDOR" ]; then - echo "No vendor lib? Cloning..." - git clone --recursive git://github.com/mozilla/zamboni-lib.git $VENDOR -fi - -# Update the vendor lib. -echo "Updating vendor..." -git submodule --quiet foreach 'git submodule --quiet sync' -git submodule --quiet sync && git submodule update --init --recursive - -if [ -z $SET_ES_TESTS ]; then - RUN_ES_TESTS=False -else - RUN_ES_TESTS=True -fi - -cat > settings_local.py < [ # Service["mysql"], Package["python2.6"], - File["$PROJ_DIR/settings_local.py"], + File["$PROJ_DIR/settings.py"], File["$PROJ_DIR/migrations/264-locale-indexes.py"], # Exec["fetch_landfill_sql"], # Exec["load_data"] diff --git a/vagrant/manifests/classes/zamboni.pp b/vagrant/manifests/classes/zamboni.pp index cc05031b85..ebe70db6e0 100644 --- a/vagrant/manifests/classes/zamboni.pp +++ b/vagrant/manifests/classes/zamboni.pp @@ -10,7 +10,7 @@ class zamboni { exec { "create_mysql_database": command => "mysqladmin -uroot create $DB_NAME", unless => "mysql -uroot -B --skip-column-names -e 'show databases' | /bin/grep '$DB_NAME'", - require => File["$PROJ_DIR/settings_local.py"] + require => File["$PROJ_DIR/settings.py"] } exec { "grant_mysql_database": diff --git a/vagrant/manifests/classes/zamboni_plus.pp b/vagrant/manifests/classes/zamboni_plus.pp index 04214e7d94..53a7e071a0 100644 --- a/vagrant/manifests/classes/zamboni_plus.pp +++ b/vagrant/manifests/classes/zamboni_plus.pp @@ -2,9 +2,9 @@ # the base image again class zamboni_plus { - file { "$PROJ_DIR/settings_local.py": + file { "$PROJ_DIR/settings.py": ensure => file, - source => "$PROJ_DIR/docs/settings/settings_local.dev.py", + source => "$PROJ_DIR/settings.py", replace => false; } diff --git a/wsgi/mkt.wsgi b/wsgi/mkt.wsgi deleted file mode 100644 index 99a06e76dd..0000000000 --- a/wsgi/mkt.wsgi +++ /dev/null @@ -1,70 +0,0 @@ -import os -import site -import sys -from datetime import datetime - -# Tell manage that we need to pull in the default settings file. -os.environ['DJANGO_SETTINGS_MODULE'] = 'settings_local_mkt' - -# Remember when mod_wsgi loaded this file so we can track it in nagios. -wsgi_loaded = datetime.now() - -# Tell celery that we're using Django. -os.environ['CELERY_LOADER'] = 'django' - -# Add the zamboni dir to the python path so we can import manage. -wsgidir = os.path.dirname(__file__) -site.addsitedir(os.path.abspath(os.path.join(wsgidir, '../'))) - -# manage adds /apps, /lib, and /vendor to the Python path. -import manage - -import django.conf -import django.core.handlers.wsgi -import django.core.management -import django.utils - -# Do validate and activate translations like using `./manage.py runserver`. -# http://blog.dscpl.com.au/2010/03/improved-wsgi-script-for-use-with.html -django.utils.translation.activate(django.conf.settings.LANGUAGE_CODE) -utility = django.core.management.ManagementUtility() -command = utility.fetch_command('runserver') -command.validate() - -# This is what mod_wsgi runs. -django_app = django.core.handlers.wsgi.WSGIHandler() - -newrelic_ini = getattr(django.conf.settings, 'NEWRELIC_INI', None) -load_newrelic = False - -if newrelic_ini: - import newrelic.agent - try: - newrelic.agent.initialize(newrelic_ini) - load_newrelic = True - except: - import logging - startup_logger = logging.getLogger('z.startup') - startup_logger.exception('Failed to load new relic config.') - - -# Normally we could let WSGIHandler run directly, but while we're dark -# launching, we want to force the script name to be empty so we don't create -# any /z links through reverse. This fixes bug 554576. -def application(env, start_response): - if 'HTTP_X_ZEUS_DL_PT' in env: - env['SCRIPT_URL'] = env['SCRIPT_NAME'] = '' - env['wsgi.loaded'] = wsgi_loaded - env['hostname'] = django.conf.settings.HOSTNAME - env['datetime'] = str(datetime.now()) - return django_app(env, start_response) - - -if load_newrelic: - application = newrelic.agent.wsgi_application()(application) -# Uncomment this to figure out what's going on with the mod_wsgi environment. -# def application(env, start_response): -# start_response('200 OK', [('Content-Type', 'text/plain')]) -# return '\n'.join('%r: %r' % item for item in sorted(env.items())) - -# vim: ft=python diff --git a/wsgi/zamboni.wsgi b/wsgi/zamboni.wsgi index 261512da7b..7eba885163 100644 --- a/wsgi/zamboni.wsgi +++ b/wsgi/zamboni.wsgi @@ -4,7 +4,7 @@ import sys from datetime import datetime # Tell manage that we need to pull in the default settings file. -os.environ['DJANGO_SETTINGS_MODULE'] = 'settings_local' +os.environ['DJANGO_SETTINGS_MODULE'] = 'settings' # Remember when mod_wsgi loaded this file so we can track it in nagios. wsgi_loaded = datetime.now()