diff --git a/.travis.yml b/.travis.yml index 54dcf86..8d2fdf5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,8 @@ language: node_js -sudo: false +dist: trusty + +sudo: required node_js: - '4' @@ -9,6 +11,14 @@ node_js: addons: apt_packages: - graphicsmagick + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-4.8 + - mysql-server-5.6 + - mysql-client-core-5.6 + - mysql-client-5.6 notifications: email: diff --git a/lib/db/mysql/index.js b/lib/db/mysql/index.js index 1bafd68..8d60873 100644 --- a/lib/db/mysql/index.js +++ b/lib/db/mysql/index.js @@ -17,7 +17,7 @@ const REQUIRED_SQL_MODES = [ 'STRICT_ALL_TABLES', 'NO_ENGINE_SUBSTITUTION', ]; -const REQUIRED_CHARSET = 'UTF8MB4_UNICODE_CI'; +const REQUIRED_CHARSET = 'UTF8MB4_BIN'; function MysqlStore(options) { @@ -260,8 +260,16 @@ MysqlStore.prototype = { if (err) { return reject(err); } - conn._fxa_initialized = true; - return resolve(conn); + + conn.query('SET NAMES utf8mb4 COLLATE utf8mb4_bin;', function(err) { + if (err) { + return reject(err); + } + + conn._fxa_initialized = true; + return resolve(conn); + }); + }); }); }); diff --git a/lib/db/mysql/patch.js b/lib/db/mysql/patch.js index f41a906..717cc6a 100644 --- a/lib/db/mysql/patch.js +++ b/lib/db/mysql/patch.js @@ -6,5 +6,5 @@ // Update this if you add a new patch, and don't forget to update // the documentation for the current schema in ../schema.sql. -module.exports.level = 3; +module.exports.level = 4; diff --git a/lib/db/mysql/patches/patch-003-004.sql b/lib/db/mysql/patches/patch-003-004.sql new file mode 100644 index 0000000..4558c2e --- /dev/null +++ b/lib/db/mysql/patches/patch-003-004.sql @@ -0,0 +1,8 @@ +-- Update profile table to utf8mb4 + +ALTER TABLE + profile + CONVERT TO CHARACTER SET utf8mb4 + COLLATE utf8mb4_bin; + +UPDATE dbMetadata SET value = '4' WHERE name = 'schema-patch-level'; diff --git a/lib/db/mysql/patches/patch-004-003.sql b/lib/db/mysql/patches/patch-004-003.sql new file mode 100644 index 0000000..642cde8 --- /dev/null +++ b/lib/db/mysql/patches/patch-004-003.sql @@ -0,0 +1,3 @@ +-- ALTER TABLE profile CONVERT TO CHARACTER SET utf8 COLLATE utf8_unicode_ci; + +-- UPDATE dbMetadata SET value = '3' WHERE name = 'schema-patch-level'; diff --git a/lib/db/mysql/schema.sql b/lib/db/mysql/schema.sql index 78dffb3..89e9e6c 100644 --- a/lib/db/mysql/schema.sql +++ b/lib/db/mysql/schema.sql @@ -31,4 +31,4 @@ CREATE TABLE IF NOT EXISTS avatar_selected ( CREATE TABLE IF NOT EXISTS profile ( userId BINARY(16) NOT NULL PRIMARY KEY, displayName VARCHAR(256) -) ENGINE=InnoDB CHARACTER SET utf8 COLLATE utf8_unicode_ci; +) ENGINE=InnoDB CHARACTER SET utf8mb4 COLLATE utf8mb4_bin; diff --git a/lib/routes/display_name/post.js b/lib/routes/display_name/post.js index cd68f4d..58ee02e 100644 --- a/lib/routes/display_name/post.js +++ b/lib/routes/display_name/post.js @@ -15,13 +15,12 @@ const EMPTY = Object.create(null); // \u007F - ascii DEL character // \u0080-\u009F - C1 (ansi escape) control characters // \u2028-\u2029 - unicode line/paragraph separator -// \uD800-\uDFFF - non-BMP surrogate pairs // \uE000-\uF8FF - BMP private use area // \uFFF9-\uFFFF - unicode "specials" block // // We might tweak this list in future. -const ALLOWED_DISPLAY_NAME_CHARS = /^(?:[^\u0000-\u001F\u007F\u0080-\u009F\u2028-\u2029\uD800-\uDFFF\uE000-\uF8FF\uFFF9-\uFFFF])*$/; +const ALLOWED_DISPLAY_NAME_CHARS = /^(?:[^\u0000-\u001F\u007F\u0080-\u009F\u2028-\u2029\uE000-\uF8FF\uFFF9-\uFFFF])*$/; module.exports = { auth: { diff --git a/test/api.js b/test/api.js index 8101e15..f2c4610 100644 --- a/test/api.js +++ b/test/api.js @@ -941,7 +941,14 @@ describe('/display_name', function() { var NAMES = [ 'André Citroën', 'the unblinking ಠ_ಠ of ckarlof', - 'abominable ☃' + 'abominable ☃', + // emoji + '👍', + '👍🏼', + '蚋', + '鱑', + '☃ 👍 André Citroën ಠ_ಠ', + 'astral symbol 𝌆 🙀' ]; return P.resolve(NAMES).each(function(NAME) { mock.token({ @@ -971,8 +978,7 @@ describe('/display_name', function() { }); }).then(function(res) { assert.equal(res.statusCode, 200); - // Using JSON.parse() on the payload seems to break the utf8 here..? - //assert.equal(JSON.parse(res.payload).displayName, NAME); + assert.equal(JSON.parse(res.payload).displayName, NAME); assert.equal(res.result.displayName, NAME); assertSecurityHeaders(res); }); @@ -988,8 +994,7 @@ describe('/display_name', function() { 'C1 next \u0085 line', 'paragraph \u2028 separator', 'private \uE005 use \uF8FF block', - 'specials \uFFFB annotation terminator', - 'pile of \uD83D\uDCA9 lol' + 'specials \uFFFB annotation terminator' ]; return P.resolve(NAMES).each(function(NAME) { mock.token({ diff --git a/test/mysql.js b/test/mysql.js index e26b29f..8066c4e 100644 --- a/test/mysql.js +++ b/test/mysql.js @@ -37,17 +37,19 @@ describe('mysql db backend', function() { mockResponses.push([null, [{ mode: 'DUMMY_VALUE,NO_ENGINE_SUBSTITUTION' }]]); mockResponses.push([null, []]); return store.ping().then(function() { - assert.equal(capturedQueries.length, 2); + assert.equal(capturedQueries.length, 3); // The first query is checking the sql_mode. assert.equal(capturedQueries[0], 'SELECT @@sql_mode AS mode'); // The second query is to set the sql_mode. assert.equal(capturedQueries[1], 'SET SESSION sql_mode = \'DUMMY_VALUE,NO_ENGINE_SUBSTITUTION,STRICT_ALL_TABLES\''); + // The third sets utf8mb4 + assert.equal(capturedQueries[2], 'SET NAMES utf8mb4 COLLATE utf8mb4_bin;'); }).then(function() { // But re-using the connection a second time return store.ping(); }).then(function() { // Should not re-issue the strict-mode queries. - assert.equal(capturedQueries.length, 2); + assert.equal(capturedQueries.length, 3); }); });