From 7e7da6b68e7039ac3dc1d4c3b1ec47a3ee1d2601 Mon Sep 17 00:00:00 2001 From: Amri Toufali Date: Sat, 6 Aug 2022 20:54:26 -0700 Subject: [PATCH] refactor Fluent locale translations --- .env-dist | 2 + nodemon.json | 4 +- package-lock.json | 60 +++++++++++++-------------- package.json | 12 +++--- server/app-constants.js | 76 +++++++++++++++++++++++++++++++++++ server/app.js | 23 +++++++++++ server/utils/fluent.js | 89 +++++++++++++++++++++++++++++++++++++++++ 7 files changed, 229 insertions(+), 37 deletions(-) create mode 100644 server/app-constants.js create mode 100644 server/app.js create mode 100644 server/utils/fluent.js diff --git a/.env-dist b/.env-dist index 07d5be700..531d4b8ed 100755 --- a/.env-dist +++ b/.env-dist @@ -78,6 +78,8 @@ MAX_NUM_ADDRESSES=5 RECRUITMENT_BANNER_LINK= RECRUITMENT_BANNER_TEXT= +SUPPORTED_LOCALES=cak,cs,cy,da,de,el,en,en-CA,en-GB,es-AR,es-CL,es-ES,es-MX,fi,fr,fy-NL,gn,hu,kab,ia,id,it,ja,nb-NO,nl,nn-NO,pt-BR,pt-PT,ro,ru,sk,sl,sq,sv-SE,tr,uk,vi,zh-CN,zh-TW + # Locales blocked from viewing Mozilla VPN promos. Use CSV without whitespace. VPN_PROMO_BLOCKED_LOCALES=zh-CN diff --git a/nodemon.json b/nodemon.json index 8dbf58f4d..3d64e8973 100644 --- a/nodemon.json +++ b/nodemon.json @@ -2,7 +2,7 @@ "restartable": "rs", "verbose": false, "events": { - "restart": "npm run build" + "//restart": "npm run build" }, "watch": [ "./*", @@ -13,5 +13,5 @@ "public/dist/*", "tests/*" ], - "ext": "js,css,hbs,json" + "ext": "js,css,hbs,json,ftl" } \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 1a75cea40..2459f578b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,8 @@ "version": "1.0.0", "license": "MPL-2.0", "dependencies": { + "@fluent/bundle": "^0.17.1", + "@fluent/langneg": "^0.6.2", "@maxmind/geoip2-node": "^3.1.0", "@sentry/node": "5.27.2", "body-parser": "1.19.0", @@ -22,8 +24,6 @@ "express-bearer-token": "2.4.0", "express-handlebars": "5.3.1", "express-session": "1.17.1", - "fluent": "0.12.0", - "fluent-langneg": "0.2.0", "git-rev-sync": "^3.0.2", "got": "10.7.0", "helmet": "4.2.0", @@ -54,7 +54,7 @@ "stylelint-config-standard": "^26.0.0" }, "engines": { - "node": "16.x", + "node": "16.15.x", "npm": "8.x" } }, @@ -762,6 +762,24 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "dev": true }, + "node_modules/@fluent/bundle": { + "version": "0.17.1", + "resolved": "https://registry.npmjs.org/@fluent/bundle/-/bundle-0.17.1.tgz", + "integrity": "sha512-CRFNT9QcSFAeFDneTF59eyv3JXFGhIIN4boUO2y22YmsuuKLyDk+N1I/NQUYz9Ab63e6V7T6vItoZIG/2oOOuw==", + "engines": { + "node": ">=12.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@fluent/langneg": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/@fluent/langneg/-/langneg-0.6.2.tgz", + "integrity": "sha512-YF4gZ4sLYRQfctpUR2uhb5UyPUYY5n/bi3OaED/Q4awKjPjlaF8tInO3uja7pnLQcmLTURkZL7L9zxv2Z5NDwg==", + "engines": { + "node": ">=12.0.0", + "npm": ">=7.0.0" + } + }, "node_modules/@humanwhocodes/config-array": { "version": "0.9.5", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.9.5.tgz", @@ -4952,22 +4970,6 @@ "integrity": "sha512-WIWGi2L3DyTUvUrwRKgGi9TwxQMUEqPOPQBVi71R96jZXJdFskXEmf54BoZaS1kknGODoIGASGEzBUYdyMCBJg==", "dev": true }, - "node_modules/fluent": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/fluent/-/fluent-0.12.0.tgz", - "integrity": "sha512-rE5FSBv/1LoJ91suQy+dJm8vGhfq2fnzURgbC6/cfJQG/xVZn0TBeh3NVoZD9mlpryLJ37NKCKnX2u5gB4s2BQ==", - "engines": { - "node": ">=8.9.0" - } - }, - "node_modules/fluent-langneg": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/fluent-langneg/-/fluent-langneg-0.2.0.tgz", - "integrity": "sha512-C1HIOeOzu7S66xpWbmLVv+qd51oxd9ndFbkI4qZWLQMVLZTGuvhxgbcPTzmWNBMMvpvQCuOEQeNPQAzsSKaCNg==", - "engines": { - "node": ">=8.9.0" - } - }, "node_modules/for-in": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", @@ -12574,6 +12576,16 @@ } } }, + "@fluent/bundle": { + "version": "0.17.1", + "resolved": "https://registry.npmjs.org/@fluent/bundle/-/bundle-0.17.1.tgz", + "integrity": "sha512-CRFNT9QcSFAeFDneTF59eyv3JXFGhIIN4boUO2y22YmsuuKLyDk+N1I/NQUYz9Ab63e6V7T6vItoZIG/2oOOuw==" + }, + "@fluent/langneg": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/@fluent/langneg/-/langneg-0.6.2.tgz", + "integrity": "sha512-YF4gZ4sLYRQfctpUR2uhb5UyPUYY5n/bi3OaED/Q4awKjPjlaF8tInO3uja7pnLQcmLTURkZL7L9zxv2Z5NDwg==" + }, "@humanwhocodes/config-array": { "version": "0.9.5", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.9.5.tgz", @@ -15733,16 +15745,6 @@ "integrity": "sha512-WIWGi2L3DyTUvUrwRKgGi9TwxQMUEqPOPQBVi71R96jZXJdFskXEmf54BoZaS1kknGODoIGASGEzBUYdyMCBJg==", "dev": true }, - "fluent": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/fluent/-/fluent-0.12.0.tgz", - "integrity": "sha512-rE5FSBv/1LoJ91suQy+dJm8vGhfq2fnzURgbC6/cfJQG/xVZn0TBeh3NVoZD9mlpryLJ37NKCKnX2u5gB4s2BQ==" - }, - "fluent-langneg": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/fluent-langneg/-/fluent-langneg-0.2.0.tgz", - "integrity": "sha512-C1HIOeOzu7S66xpWbmLVv+qd51oxd9ndFbkI4qZWLQMVLZTGuvhxgbcPTzmWNBMMvpvQCuOEQeNPQAzsSKaCNg==" - }, "for-in": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", diff --git a/package.json b/package.json index 56868f886..967de141d 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,8 @@ "url": "https://github.com/mozilla/blurts-server/issues" }, "dependencies": { + "@fluent/bundle": "^0.17.1", + "@fluent/langneg": "^0.6.2", "@maxmind/geoip2-node": "^3.1.0", "@sentry/node": "5.27.2", "body-parser": "1.19.0", @@ -19,8 +21,6 @@ "express-bearer-token": "2.4.0", "express-handlebars": "5.3.1", "express-session": "1.17.1", - "fluent": "0.12.0", - "fluent-langneg": "0.2.0", "git-rev-sync": "^3.0.2", "got": "10.7.0", "helmet": "4.2.0", @@ -56,7 +56,8 @@ }, "homepage": "https://github.com/mozilla/blurts-server", "license": "MPL-2.0", - "main": "server.js", + "main": "server/app.js", + "type": "module", "jest": { "collectCoverageFrom": [ "**/*.js", @@ -79,7 +80,7 @@ }, "scripts": { "start": "npm run build & node server.js", - "dev": "npm run build & nodemon server.js", + "dev": "nodemon server/app.js", "build": "node esbuild.js", "db:migrate": "knex migrate:latest --knexfile db/knexfile.js", "docker:build": "docker build -t blurts-server .", @@ -95,6 +96,5 @@ "test:integration-headless-ci": "MOZ_HEADLESS=1 ERROR_SHOTS=1 wdio tests/integration/wdio.conf.js", "test:integration-docker": "MOZ_HEADLESS=1 wdio tests/integration/wdio.docker.js", "test": "npm run test:db:migrate && npm run test:tests && npm run test:coveralls" - }, - "supportedLocales": "cak,cs,cy,da,de,el,en,en-CA,en-GB,es-AR,es-CL,es-ES,es-MX,fi,fr,fy-NL,gn,hu,kab,ia,id,it,ja,nb-NO,nl,nn-NO,pt-BR,pt-PT,ro,ru,sk,sl,sq,sv-SE,tr,uk,vi,zh-CN,zh-TW" + } } diff --git a/server/app-constants.js b/server/app-constants.js new file mode 100644 index 000000000..46f790974 --- /dev/null +++ b/server/app-constants.js @@ -0,0 +1,76 @@ +import 'dotenv/config.js' + +const requiredEnvVars = [ + 'NODE_ENV', + 'SERVER_URL', + 'LOGOS_ORIGIN', + 'COOKIE_SECRET', + 'SESSION_DURATION_HOURS', + 'SMTP_URL', + 'EMAIL_FROM', + 'SES_CONFIG_SET', + 'SES_NOTIFICATION_LOG_ONLY', + 'BREACH_RESOLUTION_ENABLED', + 'FXA_ENABLED', + 'FXA_SETTINGS_URL', + 'FX_REMOTE_SETTINGS_WRITER_SERVER', + 'FX_REMOTE_SETTINGS_WRITER_USER', + 'FX_REMOTE_SETTINGS_WRITER_PASS', + 'MOZLOG_FMT', + 'MOZLOG_LEVEL', + 'OAUTH_AUTHORIZATION_URI', + 'OAUTH_TOKEN_URI', + 'OAUTH_PROFILE_URI', + 'OAUTH_CLIENT_ID', + 'OAUTH_CLIENT_SECRET', + 'HIBP_KANON_API_ROOT', + 'HIBP_KANON_API_TOKEN', + 'HIBP_API_ROOT', + 'HIBP_RELOAD_BREACHES_TIMER', + 'HIBP_THROTTLE_DELAY', + 'HIBP_THROTTLE_MAX_TRIES', + 'HIBP_NOTIFY_TOKEN', + 'DATABASE_URL', + 'PAGE_TOKEN_TIMER', + 'PRODUCT_PROMOS_ENABLED', + 'REDIS_URL', + 'SENTRY_DSN', + 'DELETE_UNVERIFIED_SUBSCRIBERS_TIMER', + 'EXPERIMENT_ACTIVE', + 'MAX_NUM_ADDRESSES', + 'SUPPORTED_LOCALES' +] + +const optionalEnvVars = [ + 'PORT', + 'RECRUITMENT_BANNER_LINK', + 'RECRUITMENT_BANNER_TEXT', + 'GEOIP_GEOLITE2_PATH', + 'GEOIP_GEOLITE2_CITY_FILENAME', + 'GEOIP_GEOLITE2_COUNTRY_FILENAME', + 'VPN_PROMO_BLOCKED_LOCALES', + 'EDUCATION_VIDEO_URL_RELAY' +] + +const AppConstants = { } + +if (!process.env.SERVER_URL && process.env.NODE_ENV === 'heroku') { + process.env.SERVER_URL = `https://${process.env.HEROKU_APP_NAME}.herokuapp.com` +} + +for (const v of requiredEnvVars) { + if (process.env[v] === undefined) { + throw new Error(`Required environment variable was not set: ${v}`) + } + AppConstants[v] = process.env[v] +} + +optionalEnvVars.forEach(key => { + if (key in process.env) AppConstants[key] = process.env[key] +}) + +AppConstants.VPN_PROMO_BLOCKED_LOCALES = AppConstants.VPN_PROMO_BLOCKED_LOCALES?.split(',') + +// AppConstants.AD_UNIT_TOTAL = fs.readdirSync(new URL('./views/partials/ad-units', import.meta.url).length + +export default Object.freeze(AppConstants) diff --git a/server/app.js b/server/app.js new file mode 100644 index 000000000..5b811715d --- /dev/null +++ b/server/app.js @@ -0,0 +1,23 @@ +import express from 'express' +import accepts from 'accepts' + +import { initFluentBundles, updateAppLocale } from './utils/fluent.js' +import defaultLayout from './views/layouts/default.js' + +const app = express() +const port = process.env.NODE_ENV !== 'development' ? 3333 : null + +await initFluentBundles() + +app.use(express.json()) +app.use((req, res, next) => { + const accept = accepts(req) + req.appLocale = updateAppLocale(accept.languages()) + next() +}) + +app.get('/', (req, res) => res.send(defaultLayout(null))) + +app.listen(port, function () { + console.log(`Server ready: listening at ${this.address().port}`) +}) diff --git a/server/utils/fluent.js b/server/utils/fluent.js new file mode 100644 index 000000000..e0da7a816 --- /dev/null +++ b/server/utils/fluent.js @@ -0,0 +1,89 @@ +import { resolve, join } from 'node:path' +import { readdirSync } from 'node:fs' +import { readFile } from 'node:fs/promises' +import { FluentBundle, FluentResource } from '@fluent/bundle' +import { negotiateLanguages } from '@fluent/langneg' +import AppConstants from '../app-constants.js' + +const supportedLocales = AppConstants.SUPPORTED_LOCALES?.split(',') +const fluentBundles = {} +let appLocale // set during a request + +/** +* Create Fluent bundles for all supported locales. +* Reads .ftl files in parallel for better server start performance. +*/ +async function initFluentBundles () { + const promises = supportedLocales.map(async locale => { + const bundle = new FluentBundle(locale, { useIsolating: false }) + const dirname = resolve(`locales/${locale}`) + + try { + const filenames = readdirSync(dirname).filter(item => item.endsWith('.ftl')) + + await Promise.all(filenames.map(async filename => { + const str = await readFile(join(dirname, filename), 'utf8') + + bundle.addResource(new FluentResource(str)) + })) + } catch (e) { + console.error('Could not read Fluent file:', e) + throw new Error(e) + } + + fluentBundles[locale] = bundle + }) + + await Promise.allSettled(promises) + + console.log('Fluent bundles created:', Object.keys(fluentBundles)) +} + +/** +* Set the locale used for translations negotiated between requested and available +* @param {Array} requestedLocales - Locales requested by client. +*/ +function updateAppLocale (requestedLocales) { + appLocale = negotiateLanguages( + requestedLocales, + supportedLocales, + { strategy: 'lookup', defaultLocale: 'en' } + ) + + return appLocale +} + +/** +* Translate a message by id +* @param {string} id - The Fluent message id. +*/ +function fluentMessage (id) { + const bundle = fluentBundles[appLocale] + + if (bundle.hasMessage(id)) return bundle.getMessage(id).value + + return id +} + +/** +* Translate and transform a message pattern +* @param {string} id - The Fluent message id. +* @param {object} args - key/value pairs corresponding to pattern in Fluent resource. +* @example +* // Given FluentResource("hello = Hello, {$name}!") +* fluentFormat (hello, {name: "Jane"}) +* // Returns "Hello, Jane!" +*/ +function fluentFormat (id, args) { + const bundle = fluentBundles[appLocale] + + if (bundle.hasMessage(id)) { + const message = bundle.getMessage(id) + + return bundle.formatPattern(message.value, args) + } + + return id +} + +export { initFluentBundles, updateAppLocale, fluentMessage, fluentFormat }