tests(smokehouse): refactor to enable Smokerider (#7284)

clients/lightrider-entry.js

@@ -24,16 +24,19 @@ const LR_PRESETS = {
  * @param {Connection} connection
  * @param {string} url
  * @param {LH.Flags} flags Lighthouse flags, including `output`
- * @param {{lrDevice?: 'desktop'|'mobile', categoryIDs?: Array<string>, logAssets: boolean}} lrOpts Options coming from Lightrider
+ * @param {{lrDevice?: 'desktop'|'mobile', categoryIDs?: Array<string>, logAssets: boolean, keepRawValues: boolean}} lrOpts Options coming from Lightrider
  * @return {Promise<string|Array<string>|void>}
  */
-async function runLighthouseInLR(connection, url, flags, {lrDevice, categoryIDs, logAssets}) {
+async function runLighthouseInLR(connection, url, flags, lrOpts) {
+  const {lrDevice, categoryIDs, logAssets, keepRawValues} = lrOpts;
+
   // Certain fixes need to kick in under LR, see https://github.com/GoogleChrome/lighthouse/issues/5839
   global.isLightRider = true;
 
   // disableStorageReset because it causes render server hang
   flags.disableStorageReset = true;
   flags.logLevel = flags.logLevel || 'info';
+
   const config = lrDevice === 'desktop' ? LR_PRESETS.desktop : LR_PRESETS.mobile;
   if (categoryIDs) {
     config.settings = config.settings || {};
@@ -50,7 +53,7 @@ async function runLighthouseInLR(connection, url, flags, {lrDevice, categoryIDs,
 
     // pre process the LHR for proto
     if (flags.output === 'json' && typeof results.report === 'string') {
-      return preprocessor.processForProto(results.report);
+      return preprocessor.processForProto(results.report, {keepRawValues});
     }
 
     return results.report;

lighthouse-cli/test/smokehouse/run-smoke.js

@@ -14,88 +14,7 @@ const log = require('lighthouse-logger');
 
 /** @param {string} str */
 const purpleify = str => `${log.purple}${str}${log.reset}`;
-const smokehouseDir = 'lighthouse-cli/test/smokehouse/';
-
-/**
- * @typedef {object} SmoketestDfn
- * @property {string} id
- * @property {string} expectations
- * @property {string} config
- * @property {string | undefined} batch
- */
-
-/** @type {Array<SmoketestDfn>} */
-const SMOKETESTS = [{
-  id: 'a11y',
-  config: smokehouseDir + 'a11y/a11y-config.js',
-  expectations: 'a11y/expectations.js',
-  batch: 'parallel-first',
-}, {
-  id: 'errors',
-  expectations: smokehouseDir + 'error-expectations.js',
-  config: smokehouseDir + 'error-config.js',
-  batch: 'errors',
-}, {
-  id: 'oopif',
-  expectations: smokehouseDir + 'oopif-expectations.js',
-  config: smokehouseDir + 'oopif-config.js',
-  batch: 'parallel-first',
-}, {
-  id: 'pwa',
-  expectations: smokehouseDir + 'pwa-expectations.js',
-  config: smokehouseDir + 'pwa-config.js',
-  batch: 'parallel-second',
-}, {
-  id: 'pwa2',
-  expectations: smokehouseDir + 'pwa2-expectations.js',
-  config: smokehouseDir + 'pwa-config.js',
-  batch: 'parallel-second',
-}, {
-  id: 'pwa3',
-  expectations: smokehouseDir + 'pwa3-expectations.js',
-  config: smokehouseDir + 'pwa-config.js',
-  batch: 'parallel-first',
-}, {
-  id: 'dbw',
-  expectations: 'dobetterweb/dbw-expectations.js',
-  config: smokehouseDir + 'dbw-config.js',
-  batch: 'parallel-second',
-}, {
-  id: 'redirects',
-  expectations: 'redirects/expectations.js',
-  config: smokehouseDir + 'redirects-config.js',
-  batch: 'parallel-first',
-}, {
-  id: 'seo',
-  expectations: 'seo/expectations.js',
-  config: smokehouseDir + 'seo-config.js',
-  batch: 'parallel-first',
-}, {
-  id: 'offline',
-  expectations: 'offline-local/offline-expectations.js',
-  config: smokehouseDir + 'offline-config.js',
-  batch: 'offline',
-}, {
-  id: 'byte',
-  expectations: 'byte-efficiency/expectations.js',
-  config: smokehouseDir + 'byte-config.js',
-  batch: 'perf-opportunity',
-}, {
-  id: 'perf',
-  expectations: 'perf/expectations.js',
-  config: 'lighthouse-core/config/perf-config.js',
-  batch: 'perf-metric',
-}, {
-  id: 'lantern',
-  expectations: 'perf/lantern-expectations.js',
-  config: smokehouseDir + 'lantern-config.js',
-  batch: 'parallel-first',
-}, {
-  id: 'metrics',
-  expectations: 'tricky-metrics/expectations.js',
-  config: 'lighthouse-core/config/perf-config.js',
-  batch: 'parallel-second',
-}];
+const SMOKETESTS = require('./smoke-test-dfns').SMOKE_TEST_DFNS;
 
 /**
  * Display smokehouse output from child process
@@ -116,7 +35,7 @@ function displaySmokehouseOutput(result) {
 /**
  * Run smokehouse in child processes for selected smoketests
  * Display output from each as soon as they finish, but resolve function when ALL are complete
- * @param {Array<SmoketestDfn>} smokes
+ * @param {Array<Smokehouse.TestDfn>} smokes
  * @return {Promise<Array<{id: string, error?: Error}>>}
  */
 async function runSmokehouse(smokes) {
@@ -149,7 +68,7 @@ async function runSmokehouse(smokes) {
 /**
  * Determine batches of smoketests to run, based on argv
  * @param {string[]} argv
- * @return {Map<string|undefined, Array<SmoketestDfn>>}
+ * @return {Map<string|undefined, Array<Smokehouse.TestDfn>>}
  */
 function getSmoketestBatches(argv) {
   let smokes = [];

lighthouse-cli/test/smokehouse/smoke-test-dfns.js

@@ -0,0 +1,133 @@
+/**
+ * @license Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const path = require('path');
+const smokehouseDir = 'lighthouse-cli/test/smokehouse/';
+
+/** @type {Array<Smokehouse.TestDfn>} */
+const SMOKE_TEST_DFNS = [{
+  id: 'a11y',
+  config: smokehouseDir + 'a11y/a11y-config.js',
+  expectations: 'a11y/expectations.js',
+  batch: 'parallel-first',
+}, {
+  id: 'errors',
+  expectations: smokehouseDir + 'error-expectations.js',
+  config: smokehouseDir + 'error-config.js',
+  batch: 'errors',
+}, {
+  id: 'oopif',
+  expectations: smokehouseDir + 'oopif-expectations.js',
+  config: smokehouseDir + 'oopif-config.js',
+  batch: 'parallel-first',
+}, {
+  id: 'pwa',
+  expectations: smokehouseDir + 'pwa-expectations.js',
+  config: smokehouseDir + 'pwa-config.js',
+  batch: 'parallel-second',
+}, {
+  id: 'pwa2',
+  expectations: smokehouseDir + 'pwa2-expectations.js',
+  config: smokehouseDir + 'pwa-config.js',
+  batch: 'parallel-second',
+}, {
+  id: 'pwa3',
+  expectations: smokehouseDir + 'pwa3-expectations.js',
+  config: smokehouseDir + 'pwa-config.js',
+  batch: 'parallel-first',
+}, {
+  id: 'dbw',
+  expectations: 'dobetterweb/dbw-expectations.js',
+  config: smokehouseDir + 'dbw-config.js',
+  batch: 'parallel-second',
+}, {
+  id: 'redirects',
+  expectations: 'redirects/expectations.js',
+  config: smokehouseDir + 'redirects-config.js',
+  batch: 'parallel-first',
+}, {
+  id: 'seo',
+  expectations: 'seo/expectations.js',
+  config: smokehouseDir + 'seo-config.js',
+  batch: 'parallel-first',
+}, {
+  id: 'offline',
+  expectations: 'offline-local/offline-expectations.js',
+  config: smokehouseDir + 'offline-config.js',
+  batch: 'offline',
+}, {
+  id: 'byte',
+  expectations: 'byte-efficiency/expectations.js',
+  config: smokehouseDir + 'byte-config.js',
+  batch: 'perf-opportunity',
+}, {
+  id: 'perf',
+  expectations: 'perf/expectations.js',
+  config: 'lighthouse-core/config/perf-config.js',
+  batch: 'perf-metric',
+}, {
+  id: 'lantern',
+  expectations: 'perf/lantern-expectations.js',
+  config: smokehouseDir + 'lantern-config.js',
+  batch: 'parallel-first',
+}, {
+  id: 'metrics',
+  expectations: 'tricky-metrics/expectations.js',
+  config: 'lighthouse-core/config/perf-config.js',
+  batch: 'parallel-second',
+}];
+
+/**
+ * Attempt to resolve a path relative to the smokehouse folder.
+ * If this fails, attempts to locate the path
+ * relative to the project root.
+ * @param {string} payloadPath
+ * @return {string}
+ */
+function resolveLocalOrProjectRoot(payloadPath) {
+  let resolved;
+  try {
+    resolved = require.resolve(__dirname + '/' + payloadPath);
+  } catch (e) {
+    const cwdPath = path.resolve(__dirname + '/../../../', payloadPath);
+    resolved = require.resolve(cwdPath);
+  }
+
+  return resolved;
+}
+
+/**
+ * @param {string} configPath
+ * @return {LH.Config.Json}
+ */
+function loadConfig(configPath) {
+  return require(configPath);
+}
+
+/**
+ * @param {string} expectationsPath
+ * @return {Smokehouse.ExpectedLHR[]}
+ */
+function loadExpectations(expectationsPath) {
+  return require(expectationsPath);
+}
+
+function getSmokeTests() {
+  return SMOKE_TEST_DFNS.map(smokeTestDfn => {
+    return {
+      id: smokeTestDfn.id,
+      config: loadConfig(resolveLocalOrProjectRoot(smokeTestDfn.config)),
+      expectations: loadExpectations(resolveLocalOrProjectRoot(smokeTestDfn.expectations)),
+      batch: smokeTestDfn.batch,
+    };
+  });
+}
+
+module.exports = {
+  SMOKE_TEST_DFNS,
+  getSmokeTests,
+};

lighthouse-cli/test/smokehouse/smokehouse-report.js

@@ -0,0 +1,233 @@
+/**
+ * @license Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+/* eslint-disable no-console */
+'use strict';
+
+const log = require('lighthouse-logger');
+
+const VERBOSE = Boolean(process.env.LH_SMOKE_VERBOSE);
+const NUMBER_REGEXP = /(?:\d|\.)+/.source;
+const OPS_REGEXP = /<=?|>=?|\+\/-/.source;
+// An optional number, optional whitespace, an operator, optional whitespace, a number.
+const NUMERICAL_EXPECTATION_REGEXP =
+  new RegExp(`^(${NUMBER_REGEXP})?\\s*(${OPS_REGEXP})\\s*(${NUMBER_REGEXP})$`);
+
+/**
+ * Checks if the actual value matches the expectation. Does not recursively search. This supports
+ *    - Greater than/less than operators, e.g. "<100", ">90"
+ *    - Regular expressions
+ *    - Strict equality
+ *
+ * @param {*} actual
+ * @param {*} expected
+ * @return {boolean}
+ */
+function matchesExpectation(actual, expected) {
+  if (typeof actual === 'number' && NUMERICAL_EXPECTATION_REGEXP.test(expected)) {
+    const parts = expected.match(NUMERICAL_EXPECTATION_REGEXP);
+    const [, prefixNumber, operator, postfixNumber] = parts;
+    switch (operator) {
+      case '>':
+        return actual > postfixNumber;
+      case '>=':
+        return actual >= postfixNumber;
+      case '<':
+        return actual < postfixNumber;
+      case '<=':
+        return actual <= postfixNumber;
+      case '+/-':
+        return Math.abs(actual - prefixNumber) <= postfixNumber;
+      default:
+        throw new Error(`unexpected operator ${operator}`);
+    }
+  } else if (typeof actual === 'string' && expected instanceof RegExp && expected.test(actual)) {
+    return true;
+  } else {
+    // Strict equality check, plus NaN equivalence.
+    return Object.is(actual, expected);
+  }
+}
+
+/**
+ * Walk down expected result, comparing to actual result. If a difference is found,
+ * the path to the difference is returned, along with the expected primitive value
+ * and the value actually found at that location. If no difference is found, returns
+ * null.
+ *
+ * Only checks own enumerable properties, not object prototypes, and will loop
+ * until the stack is exhausted, so works best with simple objects (e.g. parsed JSON).
+ * @param {string} path
+ * @param {*} actual
+ * @param {*} expected
+ * @return {(Smokehouse.Difference|null)}
+ */
+function findDifference(path, actual, expected) {
+  if (matchesExpectation(actual, expected)) {
+    return null;
+  }
+
+  // If they aren't both an object we can't recurse further, so this is the difference.
+  if (actual === null || expected === null || typeof actual !== 'object' ||
+      typeof expected !== 'object' || expected instanceof RegExp) {
+    return {
+      path,
+      actual,
+      expected,
+    };
+  }
+
+  // We only care that all expected's own properties are on actual (and not the other way around).
+  for (const key of Object.keys(expected)) {
+    // Bracket numbers, but property names requiring quotes will still be unquoted.
+    const keyAccessor = /^\d+$/.test(key) ? `[${key}]` : `.${key}`;
+    const keyPath = path + keyAccessor;
+    const expectedValue = expected[key];
+
+    if (!(key in actual)) {
+      return {path: keyPath, actual: undefined, expected: expectedValue};
+    }
+
+    const actualValue = actual[key];
+    const subDifference = findDifference(keyPath, actualValue, expectedValue);
+
+    // Break on first difference found.
+    if (subDifference) {
+      return subDifference;
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Collate results into comparisons of actual and expected scores on each audit.
+ * @param {Smokehouse.ExpectedLHR} actual
+ * @param {Smokehouse.ExpectedLHR} expected
+ * @return {Smokehouse.LHRComparison}
+ */
+function collateResults(actual, expected) {
+  const auditNames = Object.keys(expected.audits);
+  const collatedAudits = auditNames.map(auditName => {
+    const actualResult = actual.audits[auditName];
+    if (!actualResult) {
+      throw new Error(`Config did not trigger run of expected audit ${auditName}`);
+    }
+
+    const expectedResult = expected.audits[auditName];
+    const diff = findDifference(auditName, actualResult, expectedResult);
+
+    return {
+      category: auditName,
+      actual: actualResult,
+      expected: expectedResult,
+      equal: !diff,
+      diff,
+    };
+  });
+
+  return {
+    audits: collatedAudits,
+    errorCode: {
+      category: 'error code',
+      actual: actual.errorCode,
+      expected: expected.errorCode,
+      equal: actual.errorCode === expected.errorCode,
+    },
+    finalUrl: {
+      category: 'final url',
+      actual: actual.finalUrl,
+      expected: expected.finalUrl,
+      equal: actual.finalUrl === expected.finalUrl,
+    },
+  };
+}
+
+/**
+ * @param {unknown} obj
+ */
+function isPlainObject(obj) {
+  return Object.prototype.toString.call(obj) === '[object Object]';
+}
+
+/**
+ * Log the result of an assertion of actual and expected results.
+ * @param {Smokehouse.Comparison} assertion
+ */
+function reportAssertion(assertion) {
+  // @ts-ignore - this doesn't exist now but could one day, so try not to break the future
+  const _toJSON = RegExp.prototype.toJSON;
+  // @ts-ignore
+  // eslint-disable-next-line no-extend-native
+  RegExp.prototype.toJSON = RegExp.prototype.toString;
+
+  if (assertion.equal) {
+    if (isPlainObject(assertion.actual)) {
+      console.log(`  ${log.greenify(log.tick)} ${assertion.category}`);
+    } else {
+      console.log(`  ${log.greenify(log.tick)} ${assertion.category}: ` +
+          log.greenify(assertion.actual));
+    }
+  } else {
+    if (assertion.diff) {
+      const diff = assertion.diff;
+      const fullActual = JSON.stringify(assertion.actual, null, 2).replace(/\n/g, '\n      ');
+      const msg = `
+  ${log.redify(log.cross)} difference at ${log.bold}${diff.path}${log.reset}
+              expected: ${JSON.stringify(diff.expected)}
+                 found: ${JSON.stringify(diff.actual)}
+
+          found result:
+      ${log.redify(fullActual)}
+`;
+      console.log(msg);
+    } else {
+      console.log(`  ${log.redify(log.cross)} ${assertion.category}:
+              expected: ${JSON.stringify(assertion.expected)}
+                 found: ${JSON.stringify(assertion.actual)}
+`);
+    }
+  }
+
+  // @ts-ignore
+  // eslint-disable-next-line no-extend-native
+  RegExp.prototype.toJSON = _toJSON;
+}
+
+/**
+ * Log all the comparisons between actual and expected test results, then print
+ * summary. Returns count of passed and failed tests.
+ * @param {Smokehouse.LHRComparison} results
+ * @return {{passed: number, failed: number}}
+ */
+function report(results) {
+  let correctCount = 0;
+  let failedCount = 0;
+
+  [results.finalUrl, results.errorCode, ...results.audits].forEach(auditAssertion => {
+    if (auditAssertion.equal) {
+      correctCount++;
+    } else {
+      failedCount++;
+    }
+
+    if (!auditAssertion.equal || VERBOSE) {
+      reportAssertion(auditAssertion);
+    }
+  });
+
+  const plural = correctCount === 1 ? '' : 's';
+  const correctStr = `${correctCount} assertion${plural}`;
+  const colorFn = correctCount === 0 ? log.redify : log.greenify;
+  console.log(`  Correctly passed ${colorFn(correctStr)}\n`);
+
+  return {
+    passed: correctCount,
+    failed: failedCount,
+  };
+}
+
+module.exports.collateResults = collateResults;
+module.exports.report = report;

lighthouse-cli/test/smokehouse/smokehouse.js

@@ -6,22 +6,6 @@
  */
 'use strict';
 
-/**
- * @typedef {{path: string, actual: *, expected: *}} Difference
- */
-
-/**
- * @typedef {{category: string, actual: *, expected: *, equal: boolean, diff?: Difference | null}} Comparison
- */
-
-/**
- * @typedef {Pick<LH.Result, 'audits' | 'finalUrl' | 'requestedUrl'> & {errorCode?: string}} ExpectedLHR
- */
-
-/**
- * @typedef {{audits: Comparison[], errorCode: Comparison, finalUrl: Comparison}} LHRComparison
- */
-
 /* eslint-disable no-console */
 
 const fs = require('fs');
@@ -29,17 +13,12 @@ const path = require('path');
 const spawnSync = require('child_process').spawnSync;
 const yargs = require('yargs');
 const log = require('lighthouse-logger');
+const {collateResults, report} = require('./smokehouse-report');
 
 const PROTOCOL_TIMEOUT_EXIT_CODE = 67;
 const PAGE_HUNG_EXIT_CODE = 68;
 const INSECURE_DOCUMENT_REQUEST_EXIT_CODE = 69;
 const RETRIES = 3;
-const VERBOSE = Boolean(process.env.LH_SMOKE_VERBOSE);
-const NUMBER_REGEXP = /(?:\d|\.)+/.source;
-const OPS_REGEXP = /<=?|>=?|\+\/-/.source;
-// An optional number, optional whitespace, an operator, optional whitespace, a number.
-const NUMERICAL_EXPECTATION_REGEXP =
-  new RegExp(`^(${NUMBER_REGEXP})?\\s*(${OPS_REGEXP})\\s*(${NUMBER_REGEXP})$`);
 
 /**
  * Attempt to resolve a path locally. If this fails, attempts to locate the path
@@ -64,7 +43,7 @@ function resolveLocalOrCwd(payloadPath) {
  * @param {string} url
  * @param {string} configPath
  * @param {boolean=} isDebug
- * @return {ExpectedLHR}
+ * @return {Smokehouse.ExpectedLHR}
  */
 function runLighthouse(url, configPath, isDebug) {
   isDebug = isDebug || Boolean(process.env.LH_SMOKE_DEBUG);
@@ -140,220 +119,6 @@ function runLighthouse(url, configPath, isDebug) {
   return JSON.parse(lhr);
 }
 
-/**
- * Checks if the actual value matches the expectation. Does not recursively search. This supports
- *    - Greater than/less than operators, e.g. "<100", ">90"
- *    - Regular expressions
- *    - Strict equality
- *
- * @param {*} actual
- * @param {*} expected
- * @return {boolean}
- */
-function matchesExpectation(actual, expected) {
-  if (typeof actual === 'number' && NUMERICAL_EXPECTATION_REGEXP.test(expected)) {
-    const parts = expected.match(NUMERICAL_EXPECTATION_REGEXP);
-    const [, prefixNumber, operator, postfixNumber] = parts;
-    switch (operator) {
-      case '>':
-        return actual > postfixNumber;
-      case '>=':
-        return actual >= postfixNumber;
-      case '<':
-        return actual < postfixNumber;
-      case '<=':
-        return actual <= postfixNumber;
-      case '+/-':
-        return Math.abs(actual - prefixNumber) <= postfixNumber;
-      default:
-        throw new Error(`unexpected operator ${operator}`);
-    }
-  } else if (typeof actual === 'string' && expected instanceof RegExp && expected.test(actual)) {
-    return true;
-  } else {
-    // Strict equality check, plus NaN equivalence.
-    return Object.is(actual, expected);
-  }
-}
-
-/**
- * Walk down expected result, comparing to actual result. If a difference is found,
- * the path to the difference is returned, along with the expected primitive value
- * and the value actually found at that location. If no difference is found, returns
- * null.
- *
- * Only checks own enumerable properties, not object prototypes, and will loop
- * until the stack is exhausted, so works best with simple objects (e.g. parsed JSON).
- * @param {string} path
- * @param {*} actual
- * @param {*} expected
- * @return {(Difference|null)}
- */
-function findDifference(path, actual, expected) {
-  if (matchesExpectation(actual, expected)) {
-    return null;
-  }
-
-  // If they aren't both an object we can't recurse further, so this is the difference.
-  if (actual === null || expected === null || typeof actual !== 'object' ||
-      typeof expected !== 'object' || expected instanceof RegExp) {
-    return {
-      path,
-      actual,
-      expected,
-    };
-  }
-
-  // We only care that all expected's own properties are on actual (and not the other way around).
-  for (const key of Object.keys(expected)) {
-    // Bracket numbers, but property names requiring quotes will still be unquoted.
-    const keyAccessor = /^\d+$/.test(key) ? `[${key}]` : `.${key}`;
-    const keyPath = path + keyAccessor;
-    const expectedValue = expected[key];
-
-    if (!(key in actual)) {
-      return {path: keyPath, actual: undefined, expected: expectedValue};
-    }
-
-    const actualValue = actual[key];
-    const subDifference = findDifference(keyPath, actualValue, expectedValue);
-
-    // Break on first difference found.
-    if (subDifference) {
-      return subDifference;
-    }
-  }
-
-  return null;
-}
-
-/**
- * Collate results into comparisons of actual and expected scores on each audit.
- * @param {ExpectedLHR} actual
- * @param {ExpectedLHR} expected
- * @return {LHRComparison}
- */
-function collateResults(actual, expected) {
-  const auditNames = Object.keys(expected.audits);
-  const collatedAudits = auditNames.map(auditName => {
-    const actualResult = actual.audits[auditName];
-    if (!actualResult) {
-      throw new Error(`Config did not trigger run of expected audit ${auditName}`);
-    }
-
-    const expectedResult = expected.audits[auditName];
-    const diff = findDifference(auditName, actualResult, expectedResult);
-
-    return {
-      category: auditName,
-      actual: actualResult,
-      expected: expectedResult,
-      equal: !diff,
-      diff,
-    };
-  });
-
-  return {
-    audits: collatedAudits,
-    errorCode: {
-      category: 'error code',
-      actual: actual.errorCode,
-      expected: expected.errorCode,
-      equal: actual.errorCode === expected.errorCode,
-    },
-    finalUrl: {
-      category: 'final url',
-      actual: actual.finalUrl,
-      expected: expected.finalUrl,
-      equal: actual.finalUrl === expected.finalUrl,
-    },
-  };
-}
-
-/**
- * @param {unknown} obj
- */
-function isPlainObject(obj) {
-  return Object.prototype.toString.call(obj) === '[object Object]';
-}
-
-/**
- * Log the result of an assertion of actual and expected results.
- * @param {Comparison} assertion
- */
-function reportAssertion(assertion) {
-  // @ts-ignore - this doesn't exist now but could one day, so try not to break the future
-  const _toJSON = RegExp.prototype.toJSON;
-  // @ts-ignore
-  // eslint-disable-next-line no-extend-native
-  RegExp.prototype.toJSON = RegExp.prototype.toString;
-
-  if (assertion.equal) {
-    if (isPlainObject(assertion.actual)) {
-      console.log(`  ${log.greenify(log.tick)} ${assertion.category}`);
-    } else {
-      console.log(`  ${log.greenify(log.tick)} ${assertion.category}: ` +
-          log.greenify(assertion.actual));
-    }
-  } else {
-    if (assertion.diff) {
-      const diff = assertion.diff;
-      const fullActual = JSON.stringify(assertion.actual, null, 2).replace(/\n/g, '\n      ');
-      const msg = `
-  ${log.redify(log.cross)} difference at ${log.bold}${diff.path}${log.reset}
-              expected: ${JSON.stringify(diff.expected)}
-                 found: ${JSON.stringify(diff.actual)}
-
-          found result:
-      ${log.redify(fullActual)}
-`;
-      console.log(msg);
-    } else {
-      console.log(`  ${log.redify(log.cross)} ${assertion.category}:
-              expected: ${JSON.stringify(assertion.expected)}
-                 found: ${JSON.stringify(assertion.actual)}
-`);
-    }
-  }
-
-  // @ts-ignore
-  // eslint-disable-next-line no-extend-native
-  RegExp.prototype.toJSON = _toJSON;
-}
-
-/**
- * Log all the comparisons between actual and expected test results, then print
- * summary. Returns count of passed and failed tests.
- * @param {LHRComparison} results
- * @return {{passed: number, failed: number}}
- */
-function report(results) {
-  let correctCount = 0;
-  let failedCount = 0;
-
-  [results.finalUrl, results.errorCode, ...results.audits].forEach(auditAssertion => {
-    if (auditAssertion.equal) {
-      correctCount++;
-    } else {
-      failedCount++;
-    }
-
-    if (!auditAssertion.equal || VERBOSE) {
-      reportAssertion(auditAssertion);
-    }
-  });
-
-  const plural = correctCount === 1 ? '' : 's';
-  const correctStr = `${correctCount} assertion${plural}`;
-  const colorFn = correctCount === 0 ? log.redify : log.greenify;
-  console.log(`  Correctly passed ${colorFn(correctStr)}\n`);
-
-  return {
-    passed: correctCount,
-    failed: failedCount,
-  };
-}
-
 const cli = yargs
   .help('help')
   .describe({
@@ -366,7 +131,7 @@ const cli = yargs
   .argv;
 
 const configPath = resolveLocalOrCwd(cli['config-path']);
-/** @type {ExpectedLHR[]} */
+/** @type {Smokehouse.ExpectedLHR[]} */
 const expectations = require(resolveLocalOrCwd(cli['expectations-path']));
 
 // Loop sequentially over expectations, comparing against Lighthouse run, and

lighthouse-core/lib/proto-preprocessor.js

@@ -17,8 +17,11 @@ const fs = require('fs');
 
 /**
   * @param {string} result
+  * @param {{keepRawValues?: boolean}} opts
   */
-function processForProto(result) {
+function processForProto(result, opts = {}) {
+  const {keepRawValues = false} = opts;
+
   /** @type {LH.Result} */
   const reportJson = JSON.parse(result);
 
@@ -53,8 +56,8 @@ function processForProto(result) {
           audit.scoreDisplayMode = 'notApplicable';
         }
       }
-      // Drop raw values. #6199
-      if ('rawValue' in audit) {
+      // Drop raw values. https://github.com/GoogleChrome/lighthouse/issues/6199
+      if (!keepRawValues && 'rawValue' in audit) {
         delete audit.rawValue;
       }
       // Normalize displayValue to always be a string, not an array. #6200

types/smokehouse.d.ts

@@ -0,0 +1,36 @@
+/**
+ * @license Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+
+ declare module Smokehouse {
+  export interface Difference {
+    path: string;
+    actual: any;
+    expected: any;
+  }
+
+  export interface Comparison {
+    category: string;
+    actual: any;
+    expected: any;
+    equal: boolean;
+    diff?: Difference | null;
+  }
+
+  export type ExpectedLHR = Pick<LH.Result, 'audits' | 'finalUrl' | 'requestedUrl'> & { errorCode?: string }
+
+  export interface LHRComparison {
+    audits: Comparison[];
+    errorCode: Comparison;
+    finalUrl: Comparison;
+  }
+
+  export interface TestDfn {
+    id: string;
+    expectations: string;
+    config: string;
+    batch: string;
+  }
+}