From 6f3f68779001aba0f1a6592468991db26b39c56c Mon Sep 17 00:00:00 2001 From: Christopher Grebs Date: Fri, 17 Aug 2018 16:36:25 +0200 Subject: [PATCH] Unify database configuration, enable 'read committed' transaction level Fixes #7158 --- docker-compose.yml | 2 +- .../install/deprecated/installation.rst | 2 +- src/olympia/conf/dev/settings.py | 20 ++------ src/olympia/conf/prod/settings.py | 20 ++------ src/olympia/conf/stage/settings.py | 20 ++------ src/olympia/lib/settings_base.py | 48 +++++++++++-------- 6 files changed, 46 insertions(+), 66 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f802643711..e3cef8992c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ x-env-mapping: &env environment: - CELERY_BROKER_URL=amqp://olympia:olympia@rabbitmq/olympia - CELERY_RESULT_BACKEND=redis://redis:6379/1 - - DATABASE_URL=mysql://root:@mysqld/olympia + - DATABASES_DEFAULT_URL=mysql://root:@mysqld/olympia - ELASTICSEARCH_LOCATION=elasticsearch:9200 - MEMCACHE_LOCATION=memcached:11211 - MYSQL_DATABASE=olympia diff --git a/docs/topics/install/deprecated/installation.rst b/docs/topics/install/deprecated/installation.rst index 8852f634e4..c55134850d 100644 --- a/docs/topics/install/deprecated/installation.rst +++ b/docs/topics/install/deprecated/installation.rst @@ -198,7 +198,7 @@ If you want to change settings, you can either add the database settings in your :ref:`local_settings.py` or set the environment variable ``DATABASE_URL``:: - export DATABASE_URL=mysql://:@/ + export DATABASES_DEFAULT_URL=mysql://:@/ If you've changed the user and password information, you need to grant permissions to the new user:: diff --git a/src/olympia/conf/dev/settings.py b/src/olympia/conf/dev/settings.py index 88c552eedb..4649fba315 100644 --- a/src/olympia/conf/dev/settings.py +++ b/src/olympia/conf/dev/settings.py @@ -56,22 +56,12 @@ ADDONS_PATH = NETAPP_STORAGE_ROOT + '/files' REVIEWER_ATTACHMENTS_PATH = MEDIA_ROOT + '/reviewer_attachment' -DATABASES = {} -DATABASES['default'] = env.db('DATABASES_DEFAULT_URL') -DATABASES['default']['ENGINE'] = 'django.db.backends.mysql' -# Run all views in a transaction (on master) unless they are decorated not to. -DATABASES['default']['ATOMIC_REQUESTS'] = True -# Pool our database connections up for 300 seconds -DATABASES['default']['CONN_MAX_AGE'] = 300 +DATABASES = { + 'default': get_db_config('DATABASES_DEFAULT_URL'), + 'slave': get_db_config('DATABASES_SLAVE_URL'), +} -DATABASES['slave'] = env.db('DATABASES_SLAVE_URL') -# Do not open a transaction for every view on the slave DB. -DATABASES['slave']['ATOMIC_REQUESTS'] = False -DATABASES['slave']['ENGINE'] = 'django.db.backends.mysql' -# Pool our database connections up for 300 seconds -DATABASES['slave']['CONN_MAX_AGE'] = 300 - -SERVICES_DATABASE = env.db('SERVICES_DATABASE_URL') +SERVICES_DATABASE = get_db_config('SERVICES_DATABASE_URL') SLAVE_DATABASES = ['slave'] diff --git a/src/olympia/conf/prod/settings.py b/src/olympia/conf/prod/settings.py index f24e78a876..86f49ede86 100644 --- a/src/olympia/conf/prod/settings.py +++ b/src/olympia/conf/prod/settings.py @@ -46,22 +46,12 @@ ADDONS_PATH = NETAPP_STORAGE_ROOT + '/files' REVIEWER_ATTACHMENTS_PATH = MEDIA_ROOT + '/reviewer_attachment' -DATABASES = {} -DATABASES['default'] = env.db('DATABASES_DEFAULT_URL') -DATABASES['default']['ENGINE'] = 'django.db.backends.mysql' -# Run all views in a transaction (on master) unless they are decorated not to. -DATABASES['default']['ATOMIC_REQUESTS'] = True -# Pool our database connections up for 300 seconds -DATABASES['default']['CONN_MAX_AGE'] = 300 +DATABASES = { + 'default': get_db_config('DATABASES_DEFAULT_URL'), + 'slave': get_db_config('DATABASES_SLAVE_URL'), +} -DATABASES['slave'] = env.db('DATABASES_SLAVE_URL') -# Do not open a transaction for every view on the slave DB. -DATABASES['slave']['ATOMIC_REQUESTS'] = False -DATABASES['slave']['ENGINE'] = 'django.db.backends.mysql' -# Pool our database connections up for 300 seconds -DATABASES['slave']['CONN_MAX_AGE'] = 300 - -SERVICES_DATABASE = env.db('SERVICES_DATABASE_URL') +SERVICES_DATABASE = get_db_config('SERVICES_DATABASE_URL') SLAVE_DATABASES = ['slave'] diff --git a/src/olympia/conf/stage/settings.py b/src/olympia/conf/stage/settings.py index ad7d828ba1..a0d6b446c5 100644 --- a/src/olympia/conf/stage/settings.py +++ b/src/olympia/conf/stage/settings.py @@ -55,22 +55,12 @@ ADDONS_PATH = NETAPP_STORAGE_ROOT + '/files' REVIEWER_ATTACHMENTS_PATH = MEDIA_ROOT + '/reviewer_attachment' -DATABASES = {} -DATABASES['default'] = env.db('DATABASES_DEFAULT_URL') -DATABASES['default']['ENGINE'] = 'django.db.backends.mysql' -# Run all views in a transaction (on master) unless they are decorated not to. -DATABASES['default']['ATOMIC_REQUESTS'] = True -# Pool our database connections up for 300 seconds -DATABASES['default']['CONN_MAX_AGE'] = 300 +DATABASES = { + 'default': get_db_config('DATABASES_DEFAULT_URL'), + 'slave': get_db_config('DATABASES_SLAVE_URL'), +} -DATABASES['slave'] = env.db('DATABASES_SLAVE_URL') -# Do not open a transaction for every view on the slave DB. -DATABASES['slave']['ATOMIC_REQUESTS'] = False -DATABASES['slave']['ENGINE'] = 'django.db.backends.mysql' -# Pool our database connections up for 300 seconds -DATABASES['slave']['CONN_MAX_AGE'] = 300 - -SERVICES_DATABASE = env.db('SERVICES_DATABASE_URL') +SERVICES_DATABASE = get_db_config('SERVICES_DATABASE_URL') SLAVE_DATABASES = ['slave'] diff --git a/src/olympia/lib/settings_base.py b/src/olympia/lib/settings_base.py index db5c07fb6c..31ef3644c2 100644 --- a/src/olympia/lib/settings_base.py +++ b/src/olympia/lib/settings_base.py @@ -109,29 +109,39 @@ def cors_endpoint_overrides(whitelist_endpoints): CORS_ENDPOINT_OVERRIDES = [] + +def get_db_config(environ_var): + values = env.db( + var=environ_var, + default='mysql://root:@localhost/olympia') + + values.update({ + # Run all views in a transaction unless they are decorated not to. + 'ATOMIC_REQUESTS': True, + # Pool our database connections up for 300 seconds + 'CONN_MAX_AGE': 300, + 'OPTIONS': { + 'sql_mode': 'STRICT_ALL_TABLES', + 'isolation_level': 'read committed' + }, + 'TEST': { + 'CHARSET': 'utf8', + 'COLLATION': 'utf8_general_ci' + }, + }) + + return values + + DATABASES = { - 'default': env.db(default='mysql://root:@localhost/olympia') + 'default': get_db_config('DATABASES_DEFAULT_URL'), } -DATABASES['default']['OPTIONS'] = {'sql_mode': 'STRICT_ALL_TABLES'} -DATABASES['default']['TEST'] = { - 'CHARSET': 'utf8', - 'COLLATION': 'utf8_general_ci' -} -# Run all views in a transaction unless they are decorated not to. -DATABASES['default']['ATOMIC_REQUESTS'] = True -# Pool our database connections up for 300 seconds -DATABASES['default']['CONN_MAX_AGE'] = 300 # A database to be used by the services scripts, which does not use Django. -# The settings can be copied from DATABASES, but since its not a full Django -# database connection, only some values are supported. -SERVICES_DATABASE = { - 'NAME': DATABASES['default']['NAME'], - 'USER': DATABASES['default']['USER'], - 'PASSWORD': DATABASES['default']['PASSWORD'], - 'HOST': DATABASES['default']['HOST'], - 'PORT': DATABASES['default']['PORT'], -} +# Please note that this is not a full Django database connection +# so the amount of values supported are limited. By default we are using +# the same connection as 'default' but that changes in prod/dev/stage. +SERVICES_DATABASE = get_db_config('DATABASES_DEFAULT_URL') DATABASE_ROUTERS = ('multidb.PinningMasterSlaveRouter',)