From 9e44fd843ef19b5766b2292a76aab020cfdeaa15 Mon Sep 17 00:00:00 2001 From: Brendan Kenny Date: Sun, 16 Oct 2016 21:38:17 -0700 Subject: [PATCH] add smokehouse, an end-to-end test runner (#781) --- lighthouse-cli/index.js | 3 +- lighthouse-cli/test/smokehouse/pwa-config.js | 67 ++++++ .../test/smokehouse/pwa-expectations.js | 128 +++++++++++ lighthouse-cli/test/smokehouse/smokehouse.js | 199 ++++++++++++++++++ package.json | 3 +- 5 files changed, 398 insertions(+), 2 deletions(-) create mode 100644 lighthouse-cli/test/smokehouse/pwa-config.js create mode 100644 lighthouse-cli/test/smokehouse/pwa-expectations.js create mode 100755 lighthouse-cli/test/smokehouse/smokehouse.js diff --git a/lighthouse-cli/index.js b/lighthouse-cli/index.js index e04c2e74cf..c9c55996c6 100755 --- a/lighthouse-cli/index.js +++ b/lighthouse-cli/index.js @@ -192,7 +192,8 @@ function lighthouseRun(addresses) { return lighthouse(address, flags, config) .then(results => Printer.write(results, outputMode, outputPath)) .then(results => { - if (outputMode !== 'html') { + // If pretty printing to the command line, also output the html report. + if (outputMode === Printer.OUTPUT_MODE.pretty) { const filename = './' + assetSaver.getFilenamePrefix({url: address}) + '.html'; Printer.write(results, 'html', filename); } diff --git a/lighthouse-cli/test/smokehouse/pwa-config.js b/lighthouse-cli/test/smokehouse/pwa-config.js new file mode 100644 index 0000000000..85a625ff7f --- /dev/null +++ b/lighthouse-cli/test/smokehouse/pwa-config.js @@ -0,0 +1,67 @@ +/** + * @license + * Copyright 2016 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; + +/** + * Config file for running PWA smokehouse audits. + */ +module.exports = { + passes: [{ + recordNetwork: true, + recordTrace: true, + gatherers: [ + 'url', + 'https', + 'manifest', + // https://github.com/GoogleChrome/lighthouse/issues/566 + // 'cache-contents' + ] + }, + { + passName: 'offlinePass', + recordNetwork: true, + gatherers: [ + 'service-worker', + 'offline' + ] + }, + { + gatherers: [ + 'http-redirect' + ] + }], + + audits: [ + 'is-on-https', + 'redirects-http', + 'service-worker', + 'works-offline', + 'manifest-display', + 'manifest-exists', + 'manifest-background-color', + 'manifest-theme-color', + 'manifest-icons-min-192', + 'manifest-icons-min-144', + 'manifest-name', + 'manifest-short-name', + 'manifest-start-url', + // https://github.com/GoogleChrome/lighthouse/issues/566 + // 'cache-start-url' + ], + + aggregations: [] +}; diff --git a/lighthouse-cli/test/smokehouse/pwa-expectations.js b/lighthouse-cli/test/smokehouse/pwa-expectations.js new file mode 100644 index 0000000000..1a7649040c --- /dev/null +++ b/lighthouse-cli/test/smokehouse/pwa-expectations.js @@ -0,0 +1,128 @@ +/** + * @license + * Copyright 2016 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; + +/** + * Expected Lighthouse audit values for various sites with stable(ish) PWA + * results. + */ +module.exports = [ + { + initialUrl: 'https://airhorner.com', + url: 'https://airhorner.com/', + audits: { + 'is-on-https': true, + 'redirects-http': true, + 'service-worker': true, + 'works-offline': true, + 'manifest-display': true, + 'manifest-exists': true, + 'manifest-background-color': true, + 'manifest-theme-color': true, + 'manifest-icons-min-192': true, + 'manifest-icons-min-144': true, + 'manifest-name': true, + 'manifest-short-name': true, + 'manifest-start-url': true, + // 'cache-start-url': true + } + }, + + { + initialUrl: 'https://www.chromestatus.com/', + url: 'https://www.chromestatus.com/features', + audits: { + 'is-on-https': true, + 'redirects-http': true, + 'service-worker': true, + 'works-offline': false, + 'manifest-display': true, + 'manifest-exists': true, + 'manifest-background-color': true, + 'manifest-theme-color': true, + 'manifest-icons-min-192': true, + 'manifest-icons-min-144': true, + 'manifest-name': true, + 'manifest-short-name': true, + 'manifest-start-url': true, + // 'cache-start-url': true + } + }, + + { + initialUrl: 'https://jakearchibald.github.io/svgomg/', + url: 'https://jakearchibald.github.io/svgomg/', + audits: { + 'is-on-https': true, + 'redirects-http': true, + 'service-worker': true, + 'works-offline': true, + 'manifest-display': true, + 'manifest-exists': true, + 'manifest-background-color': true, + 'manifest-theme-color': true, + 'manifest-icons-min-192': true, + 'manifest-icons-min-144': true, + 'manifest-name': true, + 'manifest-short-name': true, + 'manifest-start-url': true, + // 'cache-start-url': true + } + }, + + { + initialUrl: 'https://shop.polymer-project.org/', + url: 'https://shop.polymer-project.org/', + audits: { + 'is-on-https': true, + 'redirects-http': true, + 'service-worker': true, + 'works-offline': true, + 'manifest-display': true, + 'manifest-exists': true, + 'manifest-background-color': true, + 'manifest-theme-color': true, + 'manifest-icons-min-192': true, + 'manifest-icons-min-144': true, + 'manifest-name': true, + 'manifest-short-name': true, + 'manifest-start-url': true, + // 'cache-start-url': true + } + }, + + { + initialUrl: 'https://pwa.rocks', + url: 'https://pwa.rocks/', + audits: { + 'is-on-https': true, + 'redirects-http': true, + 'service-worker': true, + 'works-offline': true, + 'manifest-display': true, + 'manifest-exists': true, + 'manifest-background-color': true, + 'manifest-theme-color': true, + 'manifest-icons-min-192': true, + 'manifest-icons-min-144': true, + 'manifest-name': true, + 'manifest-short-name': true, + 'manifest-start-url': true, + // 'cache-start-url': true + } + } +]; diff --git a/lighthouse-cli/test/smokehouse/smokehouse.js b/lighthouse-cli/test/smokehouse/smokehouse.js new file mode 100755 index 0000000000..28ae27fe91 --- /dev/null +++ b/lighthouse-cli/test/smokehouse/smokehouse.js @@ -0,0 +1,199 @@ +#!/usr/bin/env node +/** + * @license + * Copyright 2016 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; + +const path = require('path'); +const execSync = require('child_process').execSync; +const yargs = require('yargs'); + +const DEFAULT_CONFIG_PATH = 'pwa-config'; +const DEFAULT_EXPECTATIONS_PATH = 'pwa-expectations'; + +const GREEN = '\x1B[32m'; +const RED = '\x1B[31m'; +const RESET = '\x1B[0m'; +const GREEN_CHECK = greenify('✓'); +const RED_X = redify('✘'); + +/** + * Add surrounding escape sequences to turn a string green when logged. + * @param {string} str + * @return {string} + */ +function greenify(str) { + return `${GREEN}${str}${RESET}`; +} + +/** + * Add surrounding escape sequences to turn a string red when logged. + * @param {string} str + * @return {string} + */ +function redify(str) { + return `${RED}${str}${RESET}`; +} + +/** + * Attempt to resolve a path locally. If this fails, attempts to locate the path + * relative to the current working directory. + * @param {string} payloadPath + * @return {string} + */ +function resolveLocalOrCwd(payloadPath) { + let resolved; + try { + resolved = require.resolve('./' + payloadPath); + } catch (e) { + const cwdPath = path.resolve(process.cwd(), payloadPath); + resolved = require.resolve(cwdPath); + } + + return resolved; +} + +/** + * Launch Chrome and do a full Lighthouse run. + * @param {string} url + * @param {string} configPath + * @return {!LighthouseResults} + */ +function runLighthouse(url, configPath) { + const command = `node lighthouse-cli/index.js ${url}`; + const options = [ + `--config-path=${configPath}`, + '--output=json', + '--quiet' + ].join(' '); + + const rawResults = execSync(command + ' ' + options, {encoding: 'utf8'}); + return JSON.parse(rawResults); +} + +/** + * Collate results into comparisons of actual and expected scores on each audit. + * @param {{url: string, audits: {score: boolean}}} actual + * @param {{url: string, audits: boolean}} expected + * @return {{finalUrl: !Object, audits: !Array}} + */ +function collateResults(actual, expected) { + const auditNames = Object.keys(expected.audits); + const collatedAudits = auditNames.map(auditName => { + const actualResult = actual.audits[auditName]; + if (!actualResult) { + throw new Error(`Config did not trigger run of expected audit ${auditName}`); + } + + const actualScore = actualResult.score; + const expectedScore = expected.audits[auditName]; + return { + category: auditName, + actual: actualScore, + expected: expectedScore, + equal: actualScore === expectedScore + }; + }); + + return { + finalUrl: { + category: 'final url', + actual: actual.url, + expected: expected.url, + equal: actual.url === expected.url + }, + audits: collatedAudits + }; +} + +/** + * Log the result of an assertion of actual and expected results. + * @param {{category: string, equal: boolean, actual: boolean, expected: boolean}} assertion + */ +function reportAssertion(assertion) { + if (assertion.equal) { + console.log(` ${GREEN_CHECK} ${assertion.category}: ` + + greenify(assertion.actual)); + } else { + console.log(` ${RED_X} ${assertion.category}: ` + + redify(`found ${assertion.actual}, expected ${assertion.expected}`)); + } +} + +/** + * Log all the comparisons between actual and expected test results, then print + * summary. Returns count of passed and failed tests. + * @param {{finalUrl: !Object, audits: !Array}} results + * @return {{passed: number, failed: number}} + */ +function report(results) { + reportAssertion(results.finalUrl); + + let correctCount = 0; + let failedCount = 0; + results.audits.forEach(auditAssertion => { + if (auditAssertion.equal) { + correctCount++; + } else { + failedCount++; + reportAssertion(auditAssertion); + } + }); + + const plural = correctCount === 1 ? '' : 's'; + const correctStr = `${correctCount} audit${plural}`; + const colorFn = correctCount === 0 ? redify : greenify; + console.log(` Correctly passed ${colorFn(correctStr)}\n`); + + return { + passed: correctCount, + failed: failedCount + }; +} + +const cli = yargs + .help('help') + .describe({ + 'config-path': 'The path to the config JSON file', + 'expectations-path': 'The path to the expected audit results file' + }) + .default('config-path', DEFAULT_CONFIG_PATH) + .default('expectations-path', DEFAULT_EXPECTATIONS_PATH) + .argv; + +const configPath = resolveLocalOrCwd(cli['config-path']); +const expectations = require(resolveLocalOrCwd(cli['expectations-path'])); + +// Loop sequentially over expectations, comparing against Lighthouse run, and +// reporting result. +let passingCount = 0; +let failingCount = 0; +expectations.forEach(expected => { + console.log(`Checking '${expected.initialUrl}'...`); + const results = runLighthouse(expected.initialUrl, configPath); + const collated = collateResults(results, expected); + const counts = report(collated); + passingCount += counts.passed; + failingCount += counts.failed; +}); + +if (passingCount) { + console.log(greenify(`${passingCount} passing`)); +} +if (failingCount) { + console.log(redify(`${failingCount} failing`)); + process.exit(1); +} diff --git a/package.json b/package.json index 796e1f826b..91092823ff 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,8 @@ "closure": "cd lighthouse-core && closure/closure-type-checking.js", "watch": "lighthouse-core/scripts/run-mocha.sh --watch", "chrome": "lighthouse-core/scripts/launch-chrome.sh", - "dbw": "npm run start -- --config-path=lighthouse-core/config/dobetterweb.json --mobile=false --output=html --output-path=results.html" + "dbw": "npm run start -- --config-path=lighthouse-core/config/dobetterweb.json --mobile=false --output=html --output-path=results.html", + "smokehouse": "node $__node_harmony lighthouse-cli/test/smokehouse/smokehouse.js" }, "devDependencies": { "babel-core": "^6.16.0",