Unify database configuration, enable 'read committed' transaction level

Fixes #7158

docker-compose.yml

@@ -4,7 +4,7 @@ x-env-mapping: &env
   environment:
     - CELERY_BROKER_URL=amqp://olympia:olympia@rabbitmq/olympia
     - CELERY_RESULT_BACKEND=redis://redis:6379/1
-    - DATABASE_URL=mysql://root:@mysqld/olympia
+    - DATABASES_DEFAULT_URL=mysql://root:@mysqld/olympia
     - ELASTICSEARCH_LOCATION=elasticsearch:9200
     - MEMCACHE_LOCATION=memcached:11211
     - MYSQL_DATABASE=olympia

docs/topics/install/deprecated/installation.rst

@@ -198,7 +198,7 @@ If you want to change settings, you can either add the database settings in
 your :ref:`local_settings.py<example-settings>` or set the environment variable
 ``DATABASE_URL``::
 
-    export DATABASE_URL=mysql://<user>:<password>@<hostname>/<database>
+    export DATABASES_DEFAULT_URL=mysql://<user>:<password>@<hostname>/<database>
 
 If you've changed the user and password information, you need to grant
 permissions to the new user::

src/olympia/conf/dev/settings.py

@@ -56,22 +56,12 @@ ADDONS_PATH = NETAPP_STORAGE_ROOT + '/files'
 
 REVIEWER_ATTACHMENTS_PATH = MEDIA_ROOT + '/reviewer_attachment'
 
-DATABASES = {}
-DATABASES['default'] = env.db('DATABASES_DEFAULT_URL')
-DATABASES['default']['ENGINE'] = 'django.db.backends.mysql'
-# Run all views in a transaction (on master) unless they are decorated not to.
-DATABASES['default']['ATOMIC_REQUESTS'] = True
-# Pool our database connections up for 300 seconds
-DATABASES['default']['CONN_MAX_AGE'] = 300
+DATABASES = {
+    'default': get_db_config('DATABASES_DEFAULT_URL'),
+    'slave': get_db_config('DATABASES_SLAVE_URL'),
+}
 
-DATABASES['slave'] = env.db('DATABASES_SLAVE_URL')
-# Do not open a transaction for every view on the slave DB.
-DATABASES['slave']['ATOMIC_REQUESTS'] = False
-DATABASES['slave']['ENGINE'] = 'django.db.backends.mysql'
-# Pool our database connections up for 300 seconds
-DATABASES['slave']['CONN_MAX_AGE'] = 300
-
-SERVICES_DATABASE = env.db('SERVICES_DATABASE_URL')
+SERVICES_DATABASE = get_db_config('SERVICES_DATABASE_URL')
 
 SLAVE_DATABASES = ['slave']
 

src/olympia/conf/prod/settings.py

@@ -46,22 +46,12 @@ ADDONS_PATH = NETAPP_STORAGE_ROOT + '/files'
 
 REVIEWER_ATTACHMENTS_PATH = MEDIA_ROOT + '/reviewer_attachment'
 
-DATABASES = {}
-DATABASES['default'] = env.db('DATABASES_DEFAULT_URL')
-DATABASES['default']['ENGINE'] = 'django.db.backends.mysql'
-# Run all views in a transaction (on master) unless they are decorated not to.
-DATABASES['default']['ATOMIC_REQUESTS'] = True
-# Pool our database connections up for 300 seconds
-DATABASES['default']['CONN_MAX_AGE'] = 300
+DATABASES = {
+    'default': get_db_config('DATABASES_DEFAULT_URL'),
+    'slave': get_db_config('DATABASES_SLAVE_URL'),
+}
 
-DATABASES['slave'] = env.db('DATABASES_SLAVE_URL')
-# Do not open a transaction for every view on the slave DB.
-DATABASES['slave']['ATOMIC_REQUESTS'] = False
-DATABASES['slave']['ENGINE'] = 'django.db.backends.mysql'
-# Pool our database connections up for 300 seconds
-DATABASES['slave']['CONN_MAX_AGE'] = 300
-
-SERVICES_DATABASE = env.db('SERVICES_DATABASE_URL')
+SERVICES_DATABASE = get_db_config('SERVICES_DATABASE_URL')
 
 SLAVE_DATABASES = ['slave']
 

src/olympia/conf/stage/settings.py

@@ -55,22 +55,12 @@ ADDONS_PATH = NETAPP_STORAGE_ROOT + '/files'
 
 REVIEWER_ATTACHMENTS_PATH = MEDIA_ROOT + '/reviewer_attachment'
 
-DATABASES = {}
-DATABASES['default'] = env.db('DATABASES_DEFAULT_URL')
-DATABASES['default']['ENGINE'] = 'django.db.backends.mysql'
-# Run all views in a transaction (on master) unless they are decorated not to.
-DATABASES['default']['ATOMIC_REQUESTS'] = True
-# Pool our database connections up for 300 seconds
-DATABASES['default']['CONN_MAX_AGE'] = 300
+DATABASES = {
+    'default': get_db_config('DATABASES_DEFAULT_URL'),
+    'slave': get_db_config('DATABASES_SLAVE_URL'),
+}
 
-DATABASES['slave'] = env.db('DATABASES_SLAVE_URL')
-# Do not open a transaction for every view on the slave DB.
-DATABASES['slave']['ATOMIC_REQUESTS'] = False
-DATABASES['slave']['ENGINE'] = 'django.db.backends.mysql'
-# Pool our database connections up for 300 seconds
-DATABASES['slave']['CONN_MAX_AGE'] = 300
-
-SERVICES_DATABASE = env.db('SERVICES_DATABASE_URL')
+SERVICES_DATABASE = get_db_config('SERVICES_DATABASE_URL')
 
 SLAVE_DATABASES = ['slave']
 

src/olympia/lib/settings_base.py

@@ -109,29 +109,39 @@ def cors_endpoint_overrides(whitelist_endpoints):
 
 CORS_ENDPOINT_OVERRIDES = []
 
+
+def get_db_config(environ_var):
+    values = env.db(
+        var=environ_var,
+        default='mysql://root:@localhost/olympia')
+
+    values.update({
+        # Run all views in a transaction unless they are decorated not to.
+        'ATOMIC_REQUESTS': True,
+        # Pool our database connections up for 300 seconds
+        'CONN_MAX_AGE': 300,
+        'OPTIONS': {
+            'sql_mode': 'STRICT_ALL_TABLES',
+            'isolation_level': 'read committed'
+        },
+        'TEST': {
+            'CHARSET': 'utf8',
+            'COLLATION': 'utf8_general_ci'
+        },
+    })
+
+    return values
+
+
 DATABASES = {
-    'default': env.db(default='mysql://root:@localhost/olympia')
+    'default': get_db_config('DATABASES_DEFAULT_URL'),
 }
-DATABASES['default']['OPTIONS'] = {'sql_mode': 'STRICT_ALL_TABLES'}
-DATABASES['default']['TEST'] = {
-    'CHARSET': 'utf8',
-    'COLLATION': 'utf8_general_ci'
-}
-# Run all views in a transaction unless they are decorated not to.
-DATABASES['default']['ATOMIC_REQUESTS'] = True
-# Pool our database connections up for 300 seconds
-DATABASES['default']['CONN_MAX_AGE'] = 300
 
 # A database to be used by the services scripts, which does not use Django.
-# The settings can be copied from DATABASES, but since its not a full Django
-# database connection, only some values are supported.
-SERVICES_DATABASE = {
-    'NAME': DATABASES['default']['NAME'],
-    'USER': DATABASES['default']['USER'],
-    'PASSWORD': DATABASES['default']['PASSWORD'],
-    'HOST': DATABASES['default']['HOST'],
-    'PORT': DATABASES['default']['PORT'],
-}
+# Please note that this is not a full Django database connection
+# so the amount of values supported are limited. By default we are using
+# the same connection as 'default' but that changes in prod/dev/stage.
+SERVICES_DATABASE = get_db_config('DATABASES_DEFAULT_URL')
 
 DATABASE_ROUTERS = ('multidb.PinningMasterSlaveRouter',)