use utf8mb4 everywhere; replace reset_db (#10935)

Makefile-docker

@@ -84,7 +84,7 @@ help_submake:
 .PHONY: initialize_db
 initialize_db:
 	rm -rf ./user-media/* ./tmp/*
-	$(PYTHON_COMMAND) manage.py reset_db
+	$(PYTHON_COMMAND) manage.py create_db --force
 	$(PYTHON_COMMAND) manage.py migrate --noinput --run-syncdb
 	$(PYTHON_COMMAND) manage.py loaddata initial.json
 	$(PYTHON_COMMAND) manage.py import_prod_versions
@@ -168,7 +168,7 @@ reindex:
 setup-ui-tests:
 	rm -rf ./user-media/* ./tmp/*
 	# Reset the database and fake database migrations
-	$(PYTHON_COMMAND) manage.py reset_db --noinput
+	$(PYTHON_COMMAND) manage.py create_db --force
 	$(PYTHON_COMMAND) manage.py migrate --noinput --run-syncdb
 	./schematic --fake src/olympia/migrations/
 

services/utils.py

@@ -85,7 +85,7 @@ def get_cdn_url(id, row):
 def getconn():
     db = settings.SERVICES_DATABASE
     return mysql.connect(host=db['HOST'], user=db['USER'],
-                         passwd=db['PASSWORD'], db=db['NAME'], charset='utf8')
+                         passwd=db['PASSWORD'], db=db['NAME'])
 
 
 mypool = pool.QueuePool(getconn, max_overflow=10, pool_size=5, recycle=300)

settings.py

@@ -59,7 +59,7 @@ AES_KEYS = {
 }
 
 DATABASES = {
-    'default': get_db_config('DATABASES_DEFAULT_URL', charset='utf8mb4'),
+    'default': get_db_config('DATABASES_DEFAULT_URL'),
 }
 
 # FxA config for local development only.

src/olympia/amo/management/commands/create_db.py

@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+import logging
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+
+import MySQLdb as mysql
+
+
+class Command(BaseCommand):
+    """Based on django_extension's reset_db command but simplifed and with
+    support for all character sets defined in settings."""
+
+    help = ('Creates the database for this project.')
+
+    def add_arguments(self, parser):
+        super(Command, self).add_arguments(parser)
+        parser.add_argument(
+            '--force', action='store_true',
+            help='Drops any existing database first.'
+        )
+
+    def handle(self, *args, **options):
+        """
+        Create the database.
+        """
+        db_info = settings.DATABASES.get('default')
+
+        engine = db_info.get('ENGINE').split('.')[-1]
+        if engine != 'mysql':
+            raise CommandError('create_db only supports mysql databases')
+
+        database_name = db_info.get('NAME')
+        kwargs = {
+            'user': db_info.get('USER'),
+            'passwd': db_info.get('PASSWORD'),
+            'host': db_info.get('HOST'),
+        }
+        if db_info.get('PORT'):
+            kwargs['port'] = int(db_info.get('PORT'))
+        connection = mysql.connect(**kwargs)
+
+        if options.get('force'):
+            drop_query = 'DROP DATABASE IF EXISTS `%s`' % database_name
+        else:
+            drop_query = None
+
+        character_set = db_info.get('OPTIONS').get('charset', 'utf8mb4')
+        create_query = 'CREATE DATABASE `%s` CHARACTER SET %s' % (
+            database_name, character_set)
+        if drop_query:
+            logging.info('Executing... "' + drop_query + '"')
+            connection.query(drop_query)
+        logging.info('Executing... "' + create_query + '"')
+        connection.query(create_query)
+
+        logging.info('Reset successful.')

src/olympia/conf/dev/settings.py

@@ -46,9 +46,8 @@ INBOUND_EMAIL_DOMAIN = env('INBOUND_EMAIL_DOMAIN',
                            default='addons-dev.allizom.org')
 
 DATABASES = {
-    'default': get_db_config('DATABASES_DEFAULT_URL', charset='utf8mb4'),
-    'replica': get_db_config(
-        'DATABASES_REPLICA_URL', atomic_requests=False, charset='utf8mb4'),
+    'default': get_db_config('DATABASES_DEFAULT_URL'),
+    'replica': get_db_config('DATABASES_REPLICA_URL', atomic_requests=False),
 }
 
 SERVICES_DATABASE = get_db_config('SERVICES_DATABASE_URL')

src/olympia/conf/prod/settings.py

@@ -34,10 +34,8 @@ INBOUND_EMAIL_DOMAIN = env('INBOUND_EMAIL_DOMAIN',
                            default='addons.mozilla.org')
 
 DATABASES = {
-    'default': get_db_config('DATABASES_DEFAULT_URL', charset='utf8mb4'),
-    'replica': get_db_config(
-        'DATABASES_REPLICA_URL', atomic_requests=False, charset='utf8mb4',
-    ),
+    'default': get_db_config('DATABASES_DEFAULT_URL'),
+    'replica': get_db_config('DATABASES_REPLICA_URL', atomic_requests=False),
 }
 
 SERVICES_DATABASE = get_db_config('SERVICES_DATABASE_URL')

src/olympia/conf/stage/settings.py

@@ -44,10 +44,8 @@ INBOUND_EMAIL_DOMAIN = env('INBOUND_EMAIL_DOMAIN',
                            default='addons.allizom.org')
 
 DATABASES = {
-    'default': get_db_config('DATABASES_DEFAULT_URL', charset='utf8mb4'),
-    'replica': get_db_config(
-        'DATABASES_REPLICA_URL', atomic_requests=False, charset='utf8mb4',
-    ),
+    'default': get_db_config('DATABASES_DEFAULT_URL'),
+    'replica': get_db_config('DATABASES_REPLICA_URL', atomic_requests=False),
 }
 
 SERVICES_DATABASE = get_db_config('SERVICES_DATABASE_URL')

src/olympia/lib/settings_base.py

@@ -104,9 +104,7 @@ CORS_ORIGIN_ALLOW_ALL = True
 CORS_URLS_REGEX = DRF_API_REGEX
 
 
-def get_db_config(environ_var, atomic_requests=True, charset='utf8'):
-    assert charset in ('utf8', 'utf8mb4')
-
+def get_db_config(environ_var, atomic_requests=True):
     values = env.db(
         var=environ_var,
         default='mysql://root:@localhost/olympia')
@@ -120,15 +118,13 @@ def get_db_config(environ_var, atomic_requests=True, charset='utf8'):
         'CONN_MAX_AGE': 300,
         'ENGINE': 'olympia.core.db.mysql',
         'OPTIONS': {
-            'charset': charset,
+            'charset': 'utf8mb4',
             'sql_mode': 'STRICT_ALL_TABLES',
             'isolation_level': 'read committed'
         },
         'TEST': {
-            'CHARSET': charset,
-            'COLLATION': (
-                'utf8_general_ci' if charset == 'utf8' else
-                'utf8mb4_general_ci')
+            'CHARSET': 'utf8mb4',
+            'COLLATION': 'utf8mb4_general_ci'
         },
     })