cli(extra-headers): Enable sending additional HTTP Headers via the CLI (#3732)

.gitignore

@@ -27,6 +27,7 @@ last-run-results.html
 latest-run
 
 closure-error.log
+yarn-error.log
 
 /chrome-linux/
 /chrome-win32/

lighthouse-cli/bin.js

@@ -5,7 +5,7 @@
  */
 'use strict';
 
-const existsSync = require('fs').existsSync;
+const fs = require('fs');
 const path = require('path');
 
 const commands = require('./commands/commands.js');
@@ -27,7 +27,7 @@ const askPermission = require('./sentry-prompt').askPermission;
  * @return {boolean}
  */
 function isDev() {
-  return existsSync(path.join(__dirname, '../.git'));
+  return fs.existsSync(path.join(__dirname, '../.git'));
 }
 
 // Tell user if there's a newer version of LH.
@@ -70,6 +70,15 @@ log.setLevel(cliFlags.logLevel);
 if (cliFlags.output === printer.OutputMode.json && !cliFlags.outputPath) {
   cliFlags.outputPath = 'stdout';
 }
+
+if (cliFlags.extraHeaders) {
+  if (cliFlags.extraHeaders.substr(0, 1) !== '{') {
+    cliFlags.extraHeaders = fs.readFileSync(cliFlags.extraHeaders, 'utf-8');
+  }
+
+  cliFlags.extraHeaders = JSON.parse(cliFlags.extraHeaders);
+}
+
 /**
  * @return {!Promise<(void|!LH.Results)>}
  */

lighthouse-cli/cli-flags.js

@@ -42,6 +42,12 @@ function getFlags(manualArgv) {
       .example(
           'lighthouse <url> --quiet --chrome-flags="--headless"',
           'Launch Headless Chrome, turn off logging')
+      .example(
+          'lighthouse <url> --extra-headers "{\\"Cookie\\":\\"monster=blue\\", \\"x-men\\":\\"wolverine\\"}"',
+          'Stringify\'d JSON HTTP Header key/value pairs to send in requests')
+      .example(
+          'lighthouse <url> --extra-headers=./path/to/file.json',
+          'Path to JSON file of HTTP Header key/value pairs to send in requests')
 
       // List of options
       .group(['verbose', 'quiet'], 'Logging:')
@@ -83,6 +89,7 @@ function getFlags(manualArgv) {
         'port': 'The port to use for the debugging protocol. Use 0 for a random port',
         'max-wait-for-load':
             'The timeout (in milliseconds) to wait before the page is considered done loading and the run should continue. WARNING: Very high values can lead to large traces and instability',
+        'extra-headers': 'Set extra HTTP Headers to pass with request',
       })
       // set aliases
       .alias({'gather-mode': 'G', 'audit-mode': 'A'})
@@ -108,6 +115,7 @@ function getFlags(manualArgv) {
       .choices('output', printer.getValidOutputOptions())
       // force as an array
       .array('blocked-url-patterns')
+      .string('extra-headers')
 
       // default values
       .default('chrome-flags', '')

lighthouse-cli/test/cli/index-test.js

@@ -40,5 +40,31 @@ describe('CLI Tests', function() {
     assert.ok(Array.isArray(output.traceCategories));
     assert.ok(output.traceCategories.length > 0);
   });
-});
 
+  describe('extra-headers', () => {
+    it('should exit with a error if the path is not valid', () => {
+      const ret = spawnSync('node', [indexPath, 'https://www.google.com',
+        '--extra-headers=./fixtures/extra-headers/not-found.json'], {encoding: 'utf8'});
+
+      assert.ok(ret.stderr.includes('no such file or directory'));
+      assert.equal(ret.status, 1);
+    });
+
+    it('should exit with a error if the file does not contain valid JSON', () => {
+      const ret = spawnSync('node', [indexPath, 'https://www.google.com',
+        '--extra-headers',
+        path.resolve(__dirname, '../fixtures/extra-headers/invalid.txt')], {encoding: 'utf8'});
+
+      assert.ok(ret.stderr.includes('Unexpected token'));
+      assert.equal(ret.status, 1);
+    });
+
+    it('should exit with a error if the passsed in string is not valid JSON', () => {
+      const ret = spawnSync('node', [indexPath, 'https://www.google.com',
+        '--extra-headers', '{notjson}'], {encoding: 'utf8'});
+
+      assert.ok(ret.stderr.includes('Unexpected token'));
+      assert.equal(ret.status, 1);
+    });
+  });
+});

lighthouse-cli/test/fixtures/extra-headers/invalid.txt

@@ -0,0 +1 @@
+NotJSON

lighthouse-cli/test/fixtures/extra-headers/valid.json

@@ -0,0 +1,4 @@
+{
+  "Cookie": "monster=blue",
+  "x-men": "wolverine"
+}

lighthouse-core/gather/driver.js

@@ -968,6 +968,20 @@ class Driver {
       .then(_ => this.sendCommand('Network.setCacheDisabled', {cacheDisabled: false}));
   }
 
+  /**
+   * @param {!Object} headers key/value pairs of HTTP Headers.
+   * @return {!Promise}
+   */
+  setExtraHTTPHeaders(headers) {
+    if (headers) {
+      return this.sendCommand('Network.setExtraHTTPHeaders', {
+        headers,
+      });
+    }
+
+    return Promise.resolve({});
+  }
+
   clearDataForOrigin(url) {
     const origin = new URL(url).origin;
 

lighthouse-core/gather/gather-runner.js

@@ -198,7 +198,8 @@ class GatherRunner {
         // Set request blocking before any network activity
         // No "clearing" is done at the end of the pass since blockUrlPatterns([]) will unset all if
         // neccessary at the beginning of the next pass.
-        .then(() => options.driver.blockUrlPatterns(blockedUrls));
+        .then(() => options.driver.blockUrlPatterns(blockedUrls))
+        .then(() => options.driver.setExtraHTTPHeaders(options.flags.extraHeaders));
 
     return options.config.gatherers.reduce((chain, gatherer) => {
       return chain.then(_ => {

lighthouse-core/report/v2/renderer/report-renderer.js

@@ -230,6 +230,7 @@ ReportRenderer.GroupJSON; // eslint-disable-line no-unused-expressions
  *     reportGroups: !Object<string, !ReportRenderer.GroupJSON>,
  *     runtimeConfig: {
  *       blockedUrlPatterns: !Array<string>,
+ *       extraHeaders: !Object,
  *       environment: !Array<{description: string, enabled: boolean, name: string}>
  *     }
  * }}

lighthouse-core/runner.js

@@ -400,7 +400,11 @@ class Runner {
       },
     ];
 
-    return {environment, blockedUrlPatterns: flags.blockedUrlPatterns || []};
+    return {
+      environment,
+      blockedUrlPatterns: flags.blockedUrlPatterns || [],
+      extraHeaders: flags.extraHeaders || {},
+    };
   }
 }
 

lighthouse-core/test/gather/driver-test.js

@@ -79,7 +79,8 @@ connection.sendCommand = function(command, params) {
     case 'Tracing.start':
     case 'ServiceWorker.enable':
     case 'ServiceWorker.disable':
-      return Promise.resolve();
+    case 'Network.setExtraHTTPHeaders':
+      return Promise.resolve({});
     case 'Tracing.end':
       return Promise.reject(new Error('tracing not started'));
     default:
@@ -239,6 +240,21 @@ describe('Browser Driver', () => {
           'de-dupes categories');
     });
   });
+
+  it('should send the Network.setExtraHTTPHeaders command when there are extra-headers', () => {
+    return driverStub.setExtraHTTPHeaders({
+      'Cookie': 'monster',
+      'x-men': 'wolverine',
+    }).then(() => {
+      assert.equal(sendCommandParams[0].command, 'Network.setExtraHTTPHeaders');
+    });
+  });
+
+  it('should not send the Network.setExtraHTTPHeaders command when there no extra-headers', () => {
+    return driverStub.setExtraHTTPHeaders().then(() => {
+      assert.equal(sendCommandParams[0], undefined);
+    });
+  });
 });
 
 describe('Multiple tab check', () => {

lighthouse-core/test/gather/fake-driver.js

@@ -70,4 +70,7 @@ module.exports = {
   blockUrlPatterns() {
     return Promise.resolve();
   },
+  setExtraHTTPHeaders() {
+    return Promise.resolve();
+  },
 };

lighthouse-core/test/gather/gather-runner-test.js

@@ -34,7 +34,8 @@ class TestGathererNoArtifact extends Gatherer {
 
 const fakeDriver = require('./fake-driver');
 
-function getMockedEmulationDriver(emulationFn, netThrottleFn, cpuThrottleFn, blockUrlFn) {
+function getMockedEmulationDriver(emulationFn, netThrottleFn, cpuThrottleFn,
+  blockUrlFn, extraHeadersFn) {
   const Driver = require('../../gather/driver');
   const Connection = require('../../gather/connections/connection');
   const EmulationDriver = class extends Driver {
@@ -72,6 +73,9 @@ function getMockedEmulationDriver(emulationFn, netThrottleFn, cpuThrottleFn, blo
         case 'Network.setBlockedURLs':
           fn = blockUrlFn;
           break;
+        case 'Network.setExtraHTTPHeaders':
+          fn = extraHeadersFn;
+          break;
         default:
           fn = null;
           break;
@@ -255,6 +259,7 @@ describe('GatherRunner', function() {
       cleanBrowserCaches: createCheck('calledCleanBrowserCaches'),
       clearDataForOrigin: createCheck('calledClearStorage'),
       blockUrlPatterns: asyncFunc,
+      setExtraHTTPHeaders: asyncFunc,
       getUserAgent: () => Promise.resolve('Fake user agent'),
     };
 
@@ -313,6 +318,7 @@ describe('GatherRunner', function() {
       cleanBrowserCaches: createCheck('calledCleanBrowserCaches'),
       clearDataForOrigin: createCheck('calledClearStorage'),
       blockUrlPatterns: asyncFunc,
+      setExtraHTTPHeaders: asyncFunc,
       getUserAgent: () => Promise.resolve('Fake user agent'),
     };
 
@@ -358,6 +364,41 @@ describe('GatherRunner', function() {
     }).then(() => assert.deepStrictEqual(receivedUrlPatterns, []));
   });
 
+
+  it('tells the driver to set additional http headers when extraHeaders flag is given', () => {
+    let receivedHeaders = null;
+    const driver = getMockedEmulationDriver(null, null, null, null, params => {
+      receivedHeaders = params.headers;
+    });
+    const headers = {
+      'Cookie': 'monster',
+      'x-men': 'wolverine',
+    };
+
+    return GatherRunner.beforePass({
+      driver,
+      flags: {
+        extraHeaders: headers,
+      },
+      config: {gatherers: []},
+    }).then(() => assert.deepStrictEqual(
+        receivedHeaders,
+        headers
+      ));
+  });
+
+  it('returns an empty object if a falsey value is passed in to extraHeaders', () => {
+    const driver = getMockedEmulationDriver(null, null, null, null, params => params.headers);
+
+    return GatherRunner.beforePass({
+      driver,
+      flags: {
+        extraHeaders: undefined,
+      },
+      config: {gatherers: []},
+    }).then((returnValue) => assert.deepStrictEqual(returnValue, {}));
+  });
+
   it('tells the driver to begin tracing', () => {
     let calledTrace = false;
     const driver = {
@@ -551,7 +592,6 @@ describe('GatherRunner', function() {
       });
   });
 
-
   it('loads gatherers from custom paths', () => {
     const root = path.resolve(__dirname, '../fixtures');
 

readme.md

@@ -86,6 +86,7 @@ Options:
   --disable-device-emulation    Disable Nexus 5X emulation                                                                                 [boolean]
   --disable-cpu-throttling      Disable CPU throttling                                                                    [boolean] [default: false]
   --disable-network-throttling  Disable network throttling                                                                                 [boolean]
+  --extra-headers               Set extra HTTP Headers to pass with request                                                                 [string]
 
 Examples:
   lighthouse <url> --view                                                   Opens the HTML report in a browser after the run completes
@@ -95,6 +96,8 @@ Examples:
   lighthouse <url> --disable-device-emulation --disable-network-throttling  Disable device emulation
   lighthouse <url> --chrome-flags="--window-size=412,732"                   Launch Chrome with a specific window size
   lighthouse <url> --quiet --chrome-flags="--headless"                      Launch Headless Chrome, turn off logging
+  lighthouse <url> --extra-headers "{\"Cookie\":\"monster=blue\"}"          Stringify\'d JSON HTTP Header key/value pairs to send in requests
+  lighthouse <url> --extra-headers=./path/to/file.json                      Path to JSON file of HTTP Header key/value pairs to send in requests
 
 For more information on Lighthouse, see https://developers.google.com/web/tools/lighthouse/.
 ```

typings/externs.d.ts

@@ -18,6 +18,7 @@ export interface Flags {
   logLevel: string;
   hostname: string;
   blockedUrlPatterns: string[];
+  extraHeaders: string;
   enableErrorReporting: boolean;
   listAllAudits: boolean;
   listTraceCategories: boolean;