Update dependencies, format code.

package.json

@@ -3,25 +3,25 @@
   "version": "1.0.10",
   "author": "Mozilla",
   "dependencies": {
-    "aws-sdk": "^2.287.0",
+    "aws-sdk": "^2.374.0",
     "body-parser": "^1.18.3",
-    "express": "^4.16.3",
+    "express": "^4.16.4",
-    "file-type": "^8.1.0",
+    "file-type": "^10.6.0",
-    "joi": "^13.4.0",
+    "joi": "^14.3.0",
     "mozlog": "^2.2.0",
-    "request": "^2.87.0",
+    "request": "^2.88.0",
     "uuid": "^3.3.2"
   },
   "devDependencies": {
     "audit-filter": "^0.3.0",
-    "eslint": "^4.19.1",
+    "eslint": "^5.10.0",
-    "prettier": "^1.14.0"
+    "prettier": "^1.15.3"
   },
   "license": "MPL-2.0",
   "main": "server.js",
   "repository": "mozilla/speech-proxy",
   "scripts": {
-    "format": "prettier server.js --single-quote --write",
+    "format": "prettier server.js --tab-width=2 --arrow-parens=always --trailing-comma=es5 --no-bracket-spacing --single-quote --write",
     "lint": "eslint .",
     "lint:deps": "npm audit --json | audit-filter --nsp-config=.nsprc --audit=-",
     "start": "node server",

server.js

@@ -7,7 +7,7 @@ const express = require('express');
 const bodyParser = require('body-parser');
 const cp = require('child_process');
 const mozlog = require('mozlog')({
-  app: 'speech-proxy'
+  app: 'speech-proxy',
 })('server');
 const request = require('request');
 const Joi = require('joi');
@@ -17,25 +17,26 @@ const fileType = require('file-type');
 
 const app = express();
 
-const regexUA = RegExp('^[a-zA-Z0-9-_ \t\\\/\.;:]{0,1024}$'); // eslint-disable-line
+// eslint-disable-next-line no-control-regex
+const regexUA = RegExp('^[a-zA-Z0-9-_ \t\\/.;:]{0,1024}$');
 
 const languages = (() => {
   const contents = fs.readFileSync('languages.json');
   return JSON.parse(contents.toString('utf8').toLowerCase());
 })();
 
-const configSchema =  Joi.object({
+const configSchema = Joi.object({
   asr_url: Joi.string(),
   disable_jail: Joi.boolean(),
   port: Joi.number(),
-  s3_bucket: Joi.string().optional()
+  s3_bucket: Joi.string().optional(),
 });
 
 const config = {
   asr_url: process.env.ASR_URL,
-  disable_jail: (process.env.DISABLE_DECODE_JAIL === '1'),
+  disable_jail: process.env.DISABLE_DECODE_JAIL === '1',
   port: process.env.PORT || 9001,
-  s3_bucket: process.env.S3_BUCKET
+  s3_bucket: process.env.S3_BUCKET,
 };
 
 mozlog.info('config', config);
@@ -43,7 +44,6 @@ mozlog.info('config', config);
 Joi.assert(config, configSchema);
 
 const validateHeaders = (headers) => {
-
   // validate the language
   if (headers['accept-language-stt'] !== undefined) {
     const lang_header = headers['accept-language-stt'].toLowerCase();
@@ -51,15 +51,23 @@ const validateHeaders = (headers) => {
     // if the passed language contains anything different from two (eg. pt)
     // or five (eg. pt-br) chars, or eleven (eg. cmn-Hans-CN)
     // or six (eg.fil-PH)  we deny
-    if (lang_header.length !== 2 && lang_header.length !== 5 &&
+    if (
-        lang_header.length !== 11 && lang_header.length !== 6) {
+      lang_header.length !== 2 &&
+      lang_header.length !== 5 &&
+      lang_header.length !== 11 &&
+      lang_header.length !== 6
+    ) {
       return 'accept-language-stt';
     }
 
     // if the passed language contains five chars, (eg. pt-br)
     // we try to match the exact key in the json, and if we find, we accept
-    if ((lang_header.length === 11 || lang_header.length === 5 ||
+    if (
-          lang_header.length === 6) && languages[lang_header] === undefined) {
+      (lang_header.length === 11 ||
+        lang_header.length === 5 ||
+        lang_header.length === 6) &&
+      languages[lang_header] === undefined
+    ) {
       return 'accept-language-stt';
     }
 
@@ -68,7 +76,7 @@ const validateHeaders = (headers) => {
     if (lang_header.length === 2) {
       let match_lang = false;
       for (const lang in languages) {
-        if (lang.substring(0,2) === lang_header) {
+        if (lang.substring(0, 2) === lang_header) {
           match_lang = true;
           break;
         }
@@ -80,17 +88,28 @@ const validateHeaders = (headers) => {
   }
 
   // validate storesample
-  if ((headers['store-sample'] !== undefined) &&  ((headers['store-sample'] !== '1') && (headers['store-sample'] !== '0'))) {
+  if (
+    headers['store-sample'] !== undefined &&
+    headers['store-sample'] !== '1' &&
+    headers['store-sample'] !== '0'
+  ) {
     return 'store-sample';
   }
 
   // validate storetranscription
-  if ((headers['store-transcription'] !== undefined) &&  ((headers['store-transcription'] !== '1') && (headers['store-transcription'] !== '0'))) {
+  if (
+    headers['store-transcription'] !== undefined &&
+    headers['store-transcription'] !== '1' &&
+    headers['store-transcription'] !== '0'
+  ) {
     return 'store-transcription';
   }
 
   // validate producttag
-  if ((headers['product-tag'] !== undefined) && (!regexUA.test(headers['product-tag']))) {
+  if (
+    headers['product-tag'] !== undefined &&
+    !regexUA.test(headers['product-tag'])
+  ) {
     return 'product-tag';
   }
 
@@ -98,7 +117,7 @@ const validateHeaders = (headers) => {
 };
 
 const S3 = new AWS.S3({
-  region: process.env.AWS_DEFAULT_REGION || 'us-east-1'
+  region: process.env.AWS_DEFAULT_REGION || 'us-east-1',
 });
 
 app.use((req, res, next) => {
@@ -111,7 +130,7 @@ app.use((req, res, next) => {
     method: req.method,
     path: req.originalUrl,
     referrer: req.get('Referrer'),
-    user_agent: req.get('User-Agent')
+    user_agent: req.get('User-Agent'),
   });
 
   res.once('finish', () => {
@@ -124,7 +143,7 @@ app.use((req, res, next) => {
       body: res.get('Content-Length'),
       time: Date.now() - request_start,
       referrer: req.get('Referrer'),
-      user_agent: req.get('User-Agent')
+      user_agent: req.get('User-Agent'),
     });
   });
 
@@ -158,13 +177,13 @@ app.use(function(req, res, next) {
 app.use(
   bodyParser.raw({
     limit: 1024000,
-    type: function () {
+    type: function() {
       return true;
-    }
+    },
   })
 );
 
-app.get('/__version__', function (req, res, next) {
+app.get('/__version__', function(req, res, next) {
   fs.readFile('version.json', (read_error, version) => {
     if (read_error) {
       return next(read_error);
@@ -174,44 +193,46 @@ app.get('/__version__', function (req, res, next) {
   });
 });
 
-app.get('/__lbheartbeat__', function (req, res) {
+app.get('/__lbheartbeat__', function(req, res) {
   res.json({message: 'Okay'});
 });
 
-app.get('/__heartbeat__', function (req, res) {
+app.get('/__heartbeat__', function(req, res) {
   let opusbytes = '';
   const hbfile = 'hb.raw';
-  if (fs.existsSync(hbfile)){
+  if (fs.existsSync(hbfile)) {
     opusbytes = fs.readFileSync(hbfile);
   }
   // send to the asr server
-  request({
+  request(
-    url: config.asr_url,
+    {
-    method: 'POST',
+      url: config.asr_url,
-    body: opusbytes,
+      method: 'POST',
-    headers: {'Content-Type': 'application/octet-stream'},
+      body: opusbytes,
-    qs: {'endofspeech': 'false', 'nbest': 10}
+      headers: {'Content-Type': 'application/octet-stream'},
-  }, function (asrErr, asrRes) {
+      qs: {endofspeech: 'false', nbest: 10},
-    // and send back the results to the client
+    },
-    if (asrErr) {
+    function(asrErr, asrRes) {
-      res.status(500);
+      // and send back the results to the client
-      return res.end();
+      if (asrErr) {
-    } else if (asrRes.statusCode === 200) {
+        res.status(500);
-      res.status(200);
+        return res.end();
-      return res.end();
+      } else if (asrRes.statusCode === 200) {
-    } else {
+        res.status(200);
-      res.status(500);
+        return res.end();
-      return res.end();
+      } else {
+        res.status(500);
+        return res.end();
+      }
     }
-  });
+  );
 });
 
 app.get('/', (req, res) => {
   res.json({message: 'Okay'});
 });
 
-app.post('*', function (req, res, next) {
+app.post('*', function(req, res, next) {
-
   let decodeArgs;
 
   // then we convert it from opus to raw pcm
@@ -219,7 +240,7 @@ app.post('*', function (req, res, next) {
     'firejail',
     '--profile=opusdec.profile',
     '--debug',
-    '--force'
+    '--force',
   ];
 
   const header_validation = validateHeaders(req.headers);
@@ -228,35 +249,46 @@ app.post('*', function (req, res, next) {
     // convert the headers to hex to log it
     const headers = JSON.stringify(req.headers);
     const hex = [];
-    for (let n = 0, l = headers.length; n < l; n ++) {
+    for (let n = 0, l = headers.length; n < l; n++) {
       const hexval = Number(headers.charCodeAt(n)).toString(16);
       hex.push(hexval);
     }
 
     mozlog.info('request.header.error', {
       request_id: res.locals.request_id,
-      error: hex.join('')
+      error: hex.join(''),
     });
     return res.status(400).json({message: 'Bad header:' + header_validation});
   }
 
   if (fileType(req.body) === null) {
-    return res.status(400).json({message: 'Body should be an Opus or Webm audio file'});
+    return res
-  } else if ((fileType(req.body).ext === 'webm') || (fileType(req.body).ext === '3gp')) {
+      .status(400)
+      .json({message: 'Body should be an Opus or Webm audio file'});
+  } else if (
+    fileType(req.body).ext === 'webm' ||
+    fileType(req.body).ext === '3gp'
+  ) {
     decodeArgs = [
-      'ffmpeg', '-i', '-', '-c:v', 'libvpx', '-f' , 's16le', '-ar',
+      'ffmpeg',
-      '16000', '-acodec',  'pcm_s16le', '-'
+      '-i',
+      '-',
+      '-c:v',
+      'libvpx',
+      '-f',
+      's16le',
+      '-ar',
+      '16000',
+      '-acodec',
+      'pcm_s16le',
+      '-',
     ];
   } else if (fileType(req.body).ext === 'opus') {
-    decodeArgs = [
+    decodeArgs = ['opusdec', '--rate', '16000', '-', '-'];
-      'opusdec',
-      '--rate',
-      '16000',
-      '-',
-      '-'
-    ];
   } else {
-    return res.status(400).json({message: 'Body should be an Opus or Webm audio file'});
+    return res
+      .status(400)
+      .json({message: 'Body should be an Opus or Webm audio file'});
   }
 
   let args = null;
@@ -267,9 +299,11 @@ app.post('*', function (req, res, next) {
   }
   const opusdec_start = Date.now();
   mozlog.info('request.opusdec.start', {
-    request_id: res.locals.request_id
+    request_id: res.locals.request_id,
+  });
+  const opusdec = cp.spawn(args[0], args.slice(1), {
+    stdio: ['pipe', 'pipe', 'pipe'],
   });
-  const opusdec = cp.spawn(args[0], args.slice(1), {stdio: ['pipe', 'pipe', 'pipe']});
 
   opusdec.on('error', next);
 
@@ -278,33 +312,41 @@ app.post('*', function (req, res, next) {
 
   // no-op to not fill up the buffer
   const opsdec_stderr_buf = [];
-  opusdec.stderr.on('data', function (data) {
+  opusdec.stderr.on('data', function(data) {
     opsdec_stderr_buf.push(data);
   });
 
   const key_uuid = uuid();
-  const key_base = key_uuid.slice(0,2) + '/' + key_uuid;
+  const key_base = key_uuid.slice(0, 2) + '/' + key_uuid;
 
   // assemble and store the metadata file
-  const metadata = {'language': req.headers['accept-language-stt'],
+  const metadata = {
-    'storesample': req.headers['store-sample'] !== undefined ? req.headers['store-sample'] : '1',
+    language: req.headers['accept-language-stt'],
-    'storetranscription': req.headers['store-transcription'] !== undefined ? req.headers['store-transcription'] : '1',
+    storesample:
-    'useragent': req.headers['user-agent'],
+      req.headers['store-sample'] !== undefined
-    'producttag': req.headers['product-tag']};
+        ? req.headers['store-sample']
+        : '1',
+    storetranscription:
+      req.headers['store-transcription'] !== undefined
+        ? req.headers['store-transcription']
+        : '1',
+    useragent: req.headers['user-agent'],
+    producttag: req.headers['product-tag'],
+  };
 
   if (config.s3_bucket) {
     const metadata_upload_params = {
       Body: JSON.stringify(metadata),
       Bucket: config.s3_bucket,
       ContentType: 'application/json',
-      Key: key_base + '/metadata.json'
+      Key: key_base + '/metadata.json',
     };
 
     const s3_request_start = Date.now();
 
     mozlog.info('request.s3.audio.start', {
       request_id: res.locals.request_id,
-      key: key_base + '/metadata.json'
+      key: key_base + '/metadata.json',
     });
 
     S3.putObject(metadata_upload_params, (s3_error) => {
@@ -314,7 +356,7 @@ app.post('*', function (req, res, next) {
           key: key_base + '/metadata.json',
           status: s3_error.statusCode,
           body: req.body.length,
-          time: Date.now() - s3_request_start
+          time: Date.now() - s3_request_start,
         });
         return next(s3_error);
       }
@@ -324,16 +366,16 @@ app.post('*', function (req, res, next) {
         key: key_base + '/metadata.json',
         status: 200,
         body: req.body.length,
-        time: Date.now() - s3_request_start
+        time: Date.now() - s3_request_start,
       });
     });
   }
 
-  opusdec.on('close', function (code) {
+  opusdec.on('close', function(code) {
     mozlog.info('request.opusdec.finish', {
       request_id: res.locals.request_id,
       time: Date.now() - opusdec_start,
-      stderr: Buffer.concat(opsdec_stderr_buf).toString('utf8')
+      stderr: Buffer.concat(opsdec_stderr_buf).toString('utf8'),
     });
     if (code !== 0) {
       next(new Error('opusdec exited with code %d', code));
@@ -345,14 +387,14 @@ app.post('*', function (req, res, next) {
       Body: req.body,
       Bucket: config.s3_bucket,
       ContentType: 'audio/opus',
-      Key: key_base + '/audio.opus'
+      Key: key_base + '/audio.opus',
     };
 
     const s3_request_start = Date.now();
 
     mozlog.info('request.s3.audio.start', {
       request_id: res.locals.request_id,
-      key: key_base + '/audio.opus'
+      key: key_base + '/audio.opus',
     });
 
     S3.putObject(audio_upload_params, (s3_error) => {
@@ -362,7 +404,7 @@ app.post('*', function (req, res, next) {
           key: key_base + '/audio.opus',
           status: s3_error.statusCode,
           body: req.body.length,
-          time: Date.now() - s3_request_start
+          time: Date.now() - s3_request_start,
         });
         return next(s3_error);
       }
@@ -372,7 +414,7 @@ app.post('*', function (req, res, next) {
         key: key_base + '/audio.opus',
         status: 200,
         body: req.body.length,
-        time: Date.now() - s3_request_start
+        time: Date.now() - s3_request_start,
       });
     });
   }
@@ -380,94 +422,106 @@ app.post('*', function (req, res, next) {
   const asr_request_start = Date.now();
 
   mozlog.info('request.asr.start', {
-    request_id: res.locals.request_id
+    request_id: res.locals.request_id,
   });
 
   // send to the asr server
-  request({
+  request(
-    url: config.asr_url,
+    {
-    method: 'POST',
+      url: config.asr_url,
-    body: opusdec.stdout,
+      method: 'POST',
-    headers: {'Content-Type': 'application/octet-stream', 'Accept-Language': metadata.language},
+      body: opusdec.stdout,
-    qs: {'endofspeech': 'false', 'nbest': 10}
+      headers: {
-  }, function (asrErr, asrRes, asrBody) {
+        'Content-Type': 'application/octet-stream',
-    // and send back the results to the client
+        'Accept-Language': metadata.language,
-    if (asrErr) {
+      },
-      mozlog.info('request.asr.error', {
+      qs: {endofspeech: 'false', nbest: 10},
+    },
+    function(asrErr, asrRes, asrBody) {
+      // and send back the results to the client
+      if (asrErr) {
+        mozlog.info('request.asr.error', {
+          request_id: res.locals.request_id,
+          time: Date.now() - asr_request_start,
+        });
+        return next(asrErr);
+      }
+
+      const resBody = asrBody && asrBody.toString('utf8');
+      try {
+        res.json(JSON.parse(resBody));
+      } catch (e) {
+        mozlog.info('request.asr.error', {
+          request_id: res.locals.request_id,
+          time: Date.now() - asr_request_start,
+        });
+        return res.status(500).json({error: 'Internal STT Server Error'});
+      }
+
+      mozlog.info('request.asr.finish', {
         request_id: res.locals.request_id,
-        time: Date.now() - asr_request_start
+        status: 200,
-      });
+        time: Date.now() - asr_request_start,
-      return next(asrErr);
-    }
-
-    const resBody = asrBody && asrBody.toString('utf8');
-    try {
-      res.json(JSON.parse(resBody));
-    } catch (e) {
-      mozlog.info('request.asr.error', {
-        request_id: res.locals.request_id,
-        time: Date.now() - asr_request_start
-      });
-      return res.status(500).json({error: 'Internal STT Server Error'});
-    }
-
-    mozlog.info('request.asr.finish', {
-      request_id: res.locals.request_id,
-      status: 200,
-      time: Date.now() - asr_request_start
-    });
-
-    if (config.s3_bucket && metadata.storetranscription === '1') {
-      const json_upload_params = {
-        Body: resBody,
-        Bucket: config.s3_bucket,
-        ContentType: 'application/json',
-        Key: key_base + '/transcript.json'
-      };
-
-      const s3_request_start = Date.now();
-
-      mozlog.info('request.s3.json.start', {
-        request_id: res.locals.request_id,
-        key: key_base + '/transcript.json'
       });
 
-      S3.putObject(json_upload_params, (s3_error) => {
+      if (config.s3_bucket && metadata.storetranscription === '1') {
-        if (s3_error) {
+        const json_upload_params = {
-          mozlog.info('request.s3.json.error', {
+          Body: resBody,
-            request_id: res.locals.request_id,
+          Bucket: config.s3_bucket,
-            key: key_base + '/transcript.json',
+          ContentType: 'application/json',
-            status: s3_error.statusCode,
+          Key: key_base + '/transcript.json',
-            time: Date.now() - s3_request_start
+        };
-          });
-          return next(s3_error);
-        }
 
-        mozlog.info('request.s3.json.finish', {
+        const s3_request_start = Date.now();
+
+        mozlog.info('request.s3.json.start', {
           request_id: res.locals.request_id,
           key: key_base + '/transcript.json',
-          status: 200,
-          time: Date.now() - s3_request_start
         });
-      });
+
+        S3.putObject(json_upload_params, (s3_error) => {
+          if (s3_error) {
+            mozlog.info('request.s3.json.error', {
+              request_id: res.locals.request_id,
+              key: key_base + '/transcript.json',
+              status: s3_error.statusCode,
+              time: Date.now() - s3_request_start,
+            });
+            return next(s3_error);
+          }
+
+          mozlog.info('request.s3.json.finish', {
+            request_id: res.locals.request_id,
+            key: key_base + '/transcript.json',
+            status: 200,
+            time: Date.now() - s3_request_start,
+          });
+        });
+      }
     }
-  });
+  );
 });
 
-app.use((err, req, res, next) => { // eslint-disable-line no-unused-vars
+app.use((err, req, res) => {
   mozlog.info('request.error', {
     request_id: res.locals.request_id,
-    error: err
+    error: err,
   });
 
   res.status(500).json({
-    message: err
+    message: err,
   });
 });
 
 const server = app.listen(config.port);
 mozlog.info('listen');
 
-process.on('SIGINT', () => { server.close(); });
+process.on('SIGINT', () => {
-process.on('SIGTERM', () => { server.close(); });
+  server.close();
-server.once('close', () => { mozlog.info('shutdown'); });
+});
+process.on('SIGTERM', () => {
+  server.close();
+});
+server.once('close', () => {
+  mozlog.info('shutdown');
+});