feat(db): support emoji in display name (#248) r=rfk,jrgm

.travis.yml

@@ -1,6 +1,8 @@
 language: node_js
 
-sudo: false
+dist: trusty
+
+sudo: required
 
 node_js:
   - '4'
@@ -9,6 +11,14 @@ node_js:
 addons:
   apt_packages:
     - graphicsmagick
+  apt:
+    sources:
+    - ubuntu-toolchain-r-test
+    packages:
+    - g++-4.8
+    - mysql-server-5.6
+    - mysql-client-core-5.6
+    - mysql-client-5.6
 
 notifications:
   email:

lib/db/mysql/index.js

@@ -17,7 +17,7 @@ const REQUIRED_SQL_MODES = [
   'STRICT_ALL_TABLES',
   'NO_ENGINE_SUBSTITUTION',
 ];
-const REQUIRED_CHARSET = 'UTF8MB4_UNICODE_CI';
+const REQUIRED_CHARSET = 'UTF8MB4_BIN';
 
 
 function MysqlStore(options) {
@@ -260,8 +260,16 @@ MysqlStore.prototype = {
             if (err) {
               return reject(err);
             }
-            conn._fxa_initialized = true;
-            return resolve(conn);
+
+            conn.query('SET NAMES utf8mb4 COLLATE utf8mb4_bin;', function(err) {
+              if (err) {
+                return reject(err);
+              }
+
+              conn._fxa_initialized = true;
+              return resolve(conn);
+            });
+
           });
         });
       });

lib/db/mysql/patch.js

@@ -6,5 +6,5 @@
 // Update this if you add a new patch, and don't forget to update
 // the documentation for the current schema in ../schema.sql.
 
-module.exports.level = 3;
+module.exports.level = 4;
 

lib/db/mysql/patches/patch-003-004.sql

@@ -0,0 +1,8 @@
+--  Update profile table to utf8mb4
+
+ALTER TABLE
+    profile
+    CONVERT TO CHARACTER SET utf8mb4
+    COLLATE utf8mb4_bin;
+
+UPDATE dbMetadata SET value = '4' WHERE name = 'schema-patch-level';

lib/db/mysql/patches/patch-004-003.sql

@@ -0,0 +1,3 @@
+-- ALTER TABLE profile CONVERT TO CHARACTER SET utf8 COLLATE utf8_unicode_ci;
+
+-- UPDATE dbMetadata SET value = '3' WHERE name = 'schema-patch-level';

lib/db/mysql/schema.sql

@@ -31,4 +31,4 @@ CREATE TABLE IF NOT EXISTS avatar_selected (
 CREATE TABLE IF NOT EXISTS profile (
   userId BINARY(16) NOT NULL PRIMARY KEY,
   displayName VARCHAR(256)
-) ENGINE=InnoDB CHARACTER SET utf8 COLLATE utf8_unicode_ci;
+) ENGINE=InnoDB CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;

lib/routes/display_name/post.js

@@ -15,13 +15,12 @@ const EMPTY = Object.create(null);
 //   \u007F         - ascii DEL character
 //   \u0080-\u009F  - C1 (ansi escape) control characters
 //   \u2028-\u2029  - unicode line/paragraph separator
-//   \uD800-\uDFFF  - non-BMP surrogate pairs
 //   \uE000-\uF8FF  - BMP private use area
 //   \uFFF9-\uFFFF  - unicode "specials" block
 //
 // We might tweak this list in future.
 
-const ALLOWED_DISPLAY_NAME_CHARS = /^(?:[^\u0000-\u001F\u007F\u0080-\u009F\u2028-\u2029\uD800-\uDFFF\uE000-\uF8FF\uFFF9-\uFFFF])*$/;
+const ALLOWED_DISPLAY_NAME_CHARS = /^(?:[^\u0000-\u001F\u007F\u0080-\u009F\u2028-\u2029\uE000-\uF8FF\uFFF9-\uFFFF])*$/;
 
 module.exports = {
   auth: {

test/api.js

@@ -941,7 +941,14 @@ describe('/display_name', function() {
       var NAMES = [
         'André Citroën',
         'the unblinking ಠ_ಠ of ckarlof',
-        'abominable ☃'
+        'abominable ☃',
+        // emoji
+        '👍',
+        '👍🏼',
+        '蚋',
+        '鱑',
+        '☃ 👍 André Citroën ಠ_ಠ',
+        'astral symbol 𝌆 🙀'
       ];
       return P.resolve(NAMES).each(function(NAME) {
         mock.token({
@@ -971,8 +978,7 @@ describe('/display_name', function() {
           });
         }).then(function(res) {
           assert.equal(res.statusCode, 200);
-          // Using JSON.parse() on the payload seems to break the utf8 here..?
-          //assert.equal(JSON.parse(res.payload).displayName, NAME);
+          assert.equal(JSON.parse(res.payload).displayName, NAME);
           assert.equal(res.result.displayName, NAME);
           assertSecurityHeaders(res);
         });
@@ -988,8 +994,7 @@ describe('/display_name', function() {
         'C1 next \u0085 line',
         'paragraph \u2028 separator',
         'private \uE005 use \uF8FF block',
-        'specials \uFFFB annotation terminator',
-        'pile of \uD83D\uDCA9 lol'
+        'specials \uFFFB annotation terminator'
       ];
       return P.resolve(NAMES).each(function(NAME) {
         mock.token({

test/mysql.js

@@ -37,17 +37,19 @@ describe('mysql db backend', function() {
     mockResponses.push([null, [{ mode: 'DUMMY_VALUE,NO_ENGINE_SUBSTITUTION' }]]);
     mockResponses.push([null, []]);
     return store.ping().then(function() {
-      assert.equal(capturedQueries.length, 2);
+      assert.equal(capturedQueries.length, 3);
       // The first query is checking the sql_mode.
       assert.equal(capturedQueries[0], 'SELECT @@sql_mode AS mode');
       // The second query is to set the sql_mode.
       assert.equal(capturedQueries[1], 'SET SESSION sql_mode = \'DUMMY_VALUE,NO_ENGINE_SUBSTITUTION,STRICT_ALL_TABLES\'');
+      // The third sets utf8mb4
+      assert.equal(capturedQueries[2], 'SET NAMES utf8mb4 COLLATE utf8mb4_bin;');
     }).then(function() {
       // But re-using the connection a second time
       return store.ping();
     }).then(function() {
       // Should not re-issue the strict-mode queries.
-      assert.equal(capturedQueries.length, 2);
+      assert.equal(capturedQueries.length, 3);
     });
   });