new_audit: add preconnect audit (avoid costly origin roundtrips) (#4362)

2018-04-27 00:04:52 +02:00 · 2018-04-27 00:04:52 +02:00 · 1176a639a5
--- a/lighthouse-cli/test/fixtures/perf/preload_tester.js
+++ b/lighthouse-cli/test/fixtures/perf/preload_tester.js
@ -7,3 +7,6 @@
 /* eslint-disable */

 document.write('<script src="level-2.js?delay=500"></script>')
+
+// load another origin
+fetch('http://localhost:10503/preload.html');
--- a/lighthouse-cli/test/smokehouse/perf/expectations.js
+++ b/lighthouse-cli/test/smokehouse/perf/expectations.js
@ -45,6 +45,14 @@ module.exports = [
          },
        },
      },
+      'uses-rel-preconnect': {
+        score: '<1',
+        details: {
+          items: {
+            length: 1,
+          },
+        },
+      },
    },
  },
  {
--- a/lighthouse-core/audits/audit.js
+++ b/lighthouse-core/audits/audit.js
@ -100,7 +100,7 @@ class Audit {

  /**
   * @param {Array<LH.Audit.Heading>} headings
-   * @param {Array<Object<string, string>>} results
+   * @param {Array<Object<string, string|number>>} results
   * @param {LH.Audit.DetailsRendererDetailsSummary} summary
   * @return {LH.Audit.DetailsRendererDetailsJSON}
   */
--- a/lighthouse-core/audits/uses-rel-preconnect.js
+++ b/lighthouse-core/audits/uses-rel-preconnect.js
@ -0,0 +1,152 @@
+/**
+ * @license Copyright 2018 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+
+'use strict';
+
+const Audit = require('./audit');
+const Util = require('../report/html/renderer/util');
+const UnusedBytes = require('./byte-efficiency/byte-efficiency-audit');
+// Preconnect establishes a "clean" socket. Chrome's socket manager will keep an unused socket
+// around for 10s. Meaning, the time delta between processing preconnect a request should be <10s,
+// otherwise it's wasted. We add a 5s margin so we are sure to capture all key requests.
+// @see https://github.com/GoogleChrome/lighthouse/issues/3106#issuecomment-333653747
+const PRECONNECT_SOCKET_MAX_IDLE = 15;
+
+class UsesRelPreconnectAudit extends Audit {
+  /**
+   * @return {LH.Audit.Meta}
+   */
+  static get meta() {
+    return {
+      name: 'uses-rel-preconnect',
+      description: 'Avoid multiple, costly round trips to any origin',
+      informative: true,
+      helpText:
+        'Consider adding preconnect or dns-prefetch resource hints to establish early ' +
+        `connections to important third-party origins. [Learn more](https://developers.google.com/web/fundamentals/performance/resource-prioritization#preconnect).`,
+      requiredArtifacts: ['devtoolsLogs'],
+      scoreDisplayMode: Audit.SCORING_MODES.NUMERIC,
+    };
+  }
+
+  /**
+   * Check if record has valid timing
+   * @param {!LH.WebInspector.NetworkRequest} record
+   * @return {!boolean}
+   */
+  static hasValidTiming(record) {
+    return record._timing && record._timing.connectEnd > 0 && record._timing.connectStart > 0;
+  }
+
+  /**
+   * Check is the connection is already open
+   * @param {!LH.WebInspector.NetworkRequest} record
+   * @return {!boolean}
+   */
+  static hasAlreadyConnectedToOrigin(record) {
+    return (
+      record._timing.dnsEnd - record._timing.dnsStart === 0 &&
+      record._timing.connectEnd - record._timing.connectStart === 0
+    );
+  }
+
+  /**
+   * Check is the connection has started before the socket idle time
+   * @param {!LH.WebInspector.NetworkRequest} record
+   * @param {!LH.WebInspector.NetworkRequest} mainResource
+   * @return {!boolean}
+   */
+  static socketStartTimeIsBelowThreshold(record, mainResource) {
+    return Math.max(0, record.startTime - mainResource.endTime) < PRECONNECT_SOCKET_MAX_IDLE;
+  }
+
+  /**
+   * @param {LH.Artifacts} artifacts
+   * @return {Promise<LH.Audit.Product>}
+   */
+  static async audit(artifacts) {
+    const devtoolsLogs = artifacts.devtoolsLogs[UsesRelPreconnectAudit.DEFAULT_PASS];
+    let maxWasted = 0;
+
+    const [networkRecords, mainResource] = await Promise.all([
+      artifacts.requestNetworkRecords(devtoolsLogs),
+      artifacts.requestMainResource(devtoolsLogs),
+    ]);
+
+    /** @type {Map<string, LH.WebInspector.NetworkRequest[]>}  */
+    const origins = new Map();
+    networkRecords
+      .forEach(record => {
+        if (
+          // filter out all resources where timing info was invalid
+          !UsesRelPreconnectAudit.hasValidTiming(record) ||
+          // filter out all resources that are loaded by the document
+          record.initiatorRequest() === mainResource ||
+          // filter out urls that do not have an origin (data, ...)
+          !record.parsedURL || !record.parsedURL.securityOrigin() ||
+          // filter out all resources that have the same origin
+          mainResource.parsedURL.securityOrigin() === record.parsedURL.securityOrigin() ||
+          // filter out all resources where origins are already resolved
+          UsesRelPreconnectAudit.hasAlreadyConnectedToOrigin(record) ||
+          // make sure the requests are below the PRECONNECT_SOCKET_MAX_IDLE (15s) mark
+          !UsesRelPreconnectAudit.socketStartTimeIsBelowThreshold(record, mainResource)
+        ) {
+          return;
+        }
+
+        const securityOrigin = record.parsedURL.securityOrigin();
+        const records = origins.get(securityOrigin) || [];
+        records.push(record);
+        origins.set(securityOrigin, records);
+      });
+
+    /** @type {Array<{url: string, type: 'ms', wastedMs: number}>}*/
+    let results = [];
+    origins.forEach(records => {
+      // Sometimes requests are done simultaneous and the connection has not been made
+      // chrome will try to connect for each network record, we get the first record
+      const firstRecordOfOrigin = records.reduce((firstRecord, record) => {
+        return (record.startTime < firstRecord.startTime) ? record: firstRecord;
+      });
+
+      const connectionTime =
+        firstRecordOfOrigin._timing.connectEnd - firstRecordOfOrigin._timing.dnsStart;
+      const timeBetweenMainResourceAndDnsStart =
+        firstRecordOfOrigin.startTime * 1000 -
+        mainResource.endTime * 1000 +
+        firstRecordOfOrigin._timing.dnsStart;
+      const wastedMs = Math.min(connectionTime, timeBetweenMainResourceAndDnsStart);
+      maxWasted = Math.max(wastedMs, maxWasted);
+      results.push({
+        url: firstRecordOfOrigin.parsedURL.securityOrigin(),
+        type: 'ms',
+        wastedMs: wastedMs,
+      });
+    });
+
+    results = results
+      .sort((a, b) => b.wastedMs - a.wastedMs);
+
+    const headings = [
+      {key: 'url', itemType: 'url', text: 'Origin'},
+      {key: 'wastedMs', itemType: 'ms', text: 'Potential Savings'},
+    ];
+    const summary = {wastedMs: maxWasted};
+    const details = Audit.makeTableDetails(headings, results, summary);
+
+    return {
+      score: UnusedBytes.scoreForWastedMs(maxWasted),
+      rawValue: maxWasted,
+      displayValue: Util.formatMilliseconds(maxWasted),
+      extendedInfo: {
+        value: results,
+      },
+      details,
+    };
+  }
+}
+
+module.exports = UsesRelPreconnectAudit;
--- a/lighthouse-core/config/default-config.js
+++ b/lighthouse-core/config/default-config.js
@ -98,6 +98,7 @@ module.exports = {
    'mainthread-work-breakdown',
    'bootup-time',
    'uses-rel-preload',
+    'uses-rel-preconnect',
    'font-display',
    'network-requests',
    'metrics',
@ -278,6 +279,7 @@ module.exports = {
        {id: 'uses-optimized-images', weight: 0, group: 'perf-hint'},
        {id: 'uses-webp-images', weight: 0, group: 'perf-hint'},
        {id: 'uses-text-compression', weight: 0, group: 'perf-hint'},
+        {id: 'uses-rel-preconnect', weight: 0, group: 'perf-hint'},
        {id: 'time-to-first-byte', weight: 0, group: 'perf-hint'},
        {id: 'redirects', weight: 0, group: 'perf-hint'},
        {id: 'uses-rel-preload', weight: 0, group: 'perf-hint'},
--- a/lighthouse-core/report/html/renderer/crc-details-renderer.js
+++ b/lighthouse-core/report/html/renderer/crc-details-renderer.js
@ -110,7 +110,7 @@ class CriticalRequestChainRenderer {
    const {file, hostname} = Util.parseURL(segment.node.request.url);
    const treevalEl = dom.find('.crc-node__tree-value', chainsEl);
    dom.find('.crc-node__tree-file', treevalEl).textContent = `${file}`;
-    dom.find('.crc-node__tree-hostname', treevalEl).textContent = `(${hostname})`;
+    dom.find('.crc-node__tree-hostname', treevalEl).textContent = hostname ? `(${hostname})` : '';

    if (!segment.hasChildren) {
      const span = dom.createElement('span', 'crc-node__chain-duration');
--- a/lighthouse-core/report/html/renderer/details-renderer.js
+++ b/lighthouse-core/report/html/renderer/details-renderer.js
@ -96,8 +96,8 @@ class DetailsRenderer {
    let title;
    try {
      const parsed = Util.parseURL(url);
-      displayedPath = parsed.file;
-      displayedHost = `(${parsed.hostname})`;
+      displayedPath = parsed.file === '/' ? parsed.origin : parsed.file;
+      displayedHost = parsed.file === '/' ? '' : `(${parsed.hostname})`;
      title = url;
    } catch (/** @type {!Error} */ e) {
      if (!(e instanceof TypeError)) {
--- a/lighthouse-core/report/html/renderer/util.js
+++ b/lighthouse-core/report/html/renderer/util.js
@ -163,7 +163,7 @@ class Util {
    name = name.replace(/([a-f0-9]{7})[a-f0-9]{13}[a-f0-9]*/g, `$1${ELLIPSIS}`);
    // Also elide other hash-like mixed-case strings
    name = name.replace(/([a-zA-Z0-9-_]{9})(?=.*[a-z])(?=.*[A-Z])(?=.*[0-9])[a-zA-Z0-9-_]{10,}/g,
-        `$1${ELLIPSIS}`);
+      `$1${ELLIPSIS}`);
    // Also elide long number sequences
    name = name.replace(/(\d{3})\d{6,}/g, `$1${ELLIPSIS}`);
    // Merge any adjacent ellipses
@ -185,8 +185,8 @@ class Util {
      const dotIndex = name.lastIndexOf('.');
      if (dotIndex >= 0) {
        name = name.slice(0, MAX_LENGTH - 1 - (name.length - dotIndex)) +
-            // Show file extension
-            `${ELLIPSIS}${name.slice(dotIndex)}`;
+          // Show file extension
+          `${ELLIPSIS}${name.slice(dotIndex)}`;
      } else {
        name = name.slice(0, MAX_LENGTH - 1) + ELLIPSIS;
      }
@ -196,13 +196,17 @@ class Util {
  }

  /**
-   * Split a URL into a file and hostname for easy display.
+   * Split a URL into a file, hostname and origin for easy display.
   * @param {string} url
-   * @return {{file: string, hostname: string}}
+   * @return {{file: string, hostname: string, origin: string}}
   */
  static parseURL(url) {
    const parsedUrl = new URL(url);
-    return {file: Util.getURLDisplayName(parsedUrl), hostname: parsedUrl.hostname};
+    return {
+      file: Util.getURLDisplayName(parsedUrl),
+      hostname: parsedUrl.hostname,
+      origin: parsedUrl.origin,
+    };
  }

  /**
--- a/lighthouse-core/test/audits/uses-rel-preconnect-test.js
+++ b/lighthouse-core/test/audits/uses-rel-preconnect-test.js
@ -0,0 +1,220 @@
+/**
+ * @license Copyright 2018 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+
+'use strict';
+
+/* eslint-env mocha */
+
+const UsesRelPreconnect = require('../../audits/uses-rel-preconnect.js');
+const assert = require('assert');
+
+const mainResource = {
+  url: 'https://www.example.com/',
+  parsedURL: {
+    securityOrigin: () => 'https://www.example.com',
+  },
+  endTime: 1,
+};
+
+describe('Performance: uses-rel-preconnect audit', () => {
+  it(`shouldn't suggest preconnect for same origin`, async () => {
+    const networkRecords = [
+      mainResource,
+      {
+        url: 'https://www.example.com/request',
+        parsedURL: {
+          securityOrigin: () => 'https://www.example.com',
+        },
+      },
+    ];
+    const artifacts = {
+      devtoolsLogs: {[UsesRelPreconnect.DEFAULT_PASS]: []},
+      requestNetworkRecords: () => Promise.resolve(networkRecords),
+      requestMainResource: () => Promise.resolve(mainResource),
+    };
+
+    const {score, rawValue, details} = await UsesRelPreconnect.audit(artifacts);
+    assert.equal(score, 1);
+    assert.equal(rawValue, 0);
+    assert.equal(details.items.length, 0);
+  });
+
+  it(`shouldn't suggest preconnect when initiator is main resource`, async () => {
+    const networkRecords = [
+      mainResource,
+      {
+        url: 'https://cdn.example.com/request',
+        initiatorRequest: () => mainResource,
+      },
+    ];
+    const artifacts = {
+      devtoolsLogs: {[UsesRelPreconnect.DEFAULT_PASS]: []},
+      requestNetworkRecords: () => Promise.resolve(networkRecords),
+      requestMainResource: () => Promise.resolve(mainResource),
+    };
+
+    const {score, rawValue, details} = await UsesRelPreconnect.audit(artifacts);
+    assert.equal(score, 1);
+    assert.equal(rawValue, 0);
+    assert.equal(details.items.length, 0);
+  });
+
+  it(`shouldn't suggest non http(s) protocols as preconnect`, async () => {
+    const networkRecords = [
+      mainResource,
+      {
+        url: 'data:text/plain;base64,hello',
+        initiatorRequest: () => null,
+      },
+    ];
+    const artifacts = {
+      devtoolsLogs: {[UsesRelPreconnect.DEFAULT_PASS]: []},
+      requestNetworkRecords: () => Promise.resolve(networkRecords),
+      requestMainResource: () => Promise.resolve(mainResource),
+    };
+
+    const {score, rawValue, details} = await UsesRelPreconnect.audit(artifacts);
+    assert.equal(score, 1);
+    assert.equal(rawValue, 0);
+    assert.equal(details.items.length, 0);
+  });
+
+  it(`shouldn't suggest preconnect when already connected to the origin`, async () => {
+    const networkRecords = [
+      mainResource,
+      {
+        url: 'https://cdn.example.com/request',
+        initiatorRequest: () => null,
+        _timing: {
+          dnsStart: -1,
+          dnsEnd: -1,
+          connectEnd: -1,
+          connectStart: -1,
+        },
+      },
+    ];
+    const artifacts = {
+      devtoolsLogs: {[UsesRelPreconnect.DEFAULT_PASS]: []},
+      requestNetworkRecords: () => Promise.resolve(networkRecords),
+      requestMainResource: () => Promise.resolve(mainResource),
+    };
+
+    const {score, rawValue, details} = await UsesRelPreconnect.audit(artifacts);
+    assert.equal(score, 1);
+    assert.equal(rawValue, 0);
+    assert.equal(details.items.length, 0);
+  });
+
+  it(`shouldn't suggest preconnect when request has been fired after 15s`, async () => {
+    const networkRecords = [
+      mainResource,
+      {
+        url: 'https://cdn.example.com/request',
+        initiatorRequest: () => null,
+        _startTime: 16,
+      },
+    ];
+    const artifacts = {
+      devtoolsLogs: {[UsesRelPreconnect.DEFAULT_PASS]: []},
+      requestNetworkRecords: () => Promise.resolve(networkRecords),
+      requestMainResource: () => Promise.resolve(mainResource),
+    };
+
+    const {score, rawValue, details} = await UsesRelPreconnect.audit(artifacts);
+    assert.equal(score, 1);
+    assert.equal(rawValue, 0);
+    assert.equal(details.items.length, 0);
+  });
+
+  it(`should only list an origin once`, async () => {
+    const networkRecords = [
+      mainResource,
+      {
+        url: 'https://cdn.example.com/first',
+        initiatorRequest: () => null,
+        parsedURL: {
+          securityOrigin: () => 'https://cdn.example.com',
+        },
+        startTime: 2,
+        _timing: {
+          dnsStart: 100,
+          connectStart: 150,
+          connectEnd: 300,
+        },
+      },
+      {
+        url: 'https://cdn.example.com/second',
+        initiatorRequest: () => null,
+        parsedURL: {
+          securityOrigin: () => 'https://cdn.example.com',
+        },
+        startTime: 3,
+        _timing: {
+          dnsStart: 300,
+          connectStart: 350,
+          connectEnd: 400,
+        },
+      },
+    ];
+    const artifacts = {
+      devtoolsLogs: {[UsesRelPreconnect.DEFAULT_PASS]: []},
+      requestNetworkRecords: () => Promise.resolve(networkRecords),
+      requestMainResource: () => Promise.resolve(mainResource),
+    };
+
+    const {rawValue, extendedInfo} = await UsesRelPreconnect.audit(artifacts);
+    assert.equal(rawValue, 200);
+    assert.equal(extendedInfo.value.length, 1);
+    assert.deepStrictEqual(extendedInfo.value, [
+      {url: 'https://cdn.example.com', wastedMs: 200, type: 'ms'},
+    ]);
+  });
+
+  it(`should give a list of preconnected origins`, async () => {
+    const networkRecords = [
+      mainResource,
+      {
+        url: 'https://cdn.example.com/first',
+        initiatorRequest: () => null,
+        parsedURL: {
+          securityOrigin: () => 'https://cdn.example.com',
+        },
+        startTime: 2,
+        _timing: {
+          dnsStart: 100,
+          connectStart: 250,
+          connectEnd: 300,
+        },
+      },
+      {
+        url: 'https://othercdn.example.com/second',
+        initiatorRequest: () => null,
+        parsedURL: {
+          securityOrigin: () => 'https://othercdn.example.com',
+        },
+        startTime: 1.2,
+        _timing: {
+          dnsStart: 100,
+          connectStart: 200,
+          connectEnd: 600,
+        },
+      },
+    ];
+    const artifacts = {
+      devtoolsLogs: {[UsesRelPreconnect.DEFAULT_PASS]: []},
+      requestNetworkRecords: () => Promise.resolve(networkRecords),
+      requestMainResource: () => Promise.resolve(mainResource),
+    };
+
+    const {rawValue, extendedInfo} = await UsesRelPreconnect.audit(artifacts);
+    assert.equal(rawValue, 300);
+    assert.equal(extendedInfo.value.length, 2);
+    assert.deepStrictEqual(extendedInfo.value, [
+      {url: 'https://othercdn.example.com', wastedMs: 300, type: 'ms'},
+      {url: 'https://cdn.example.com', wastedMs: 200, type: 'ms'},
+    ]);
+  });
+});
--- a/lighthouse-core/test/report/html/renderer/details-renderer-test.js
+++ b/lighthouse-core/test/report/html/renderer/details-renderer-test.js
@ -154,7 +154,7 @@ describe('DetailsRenderer', () => {

    it('renders text URLs', () => {
      const urlText = 'https://example.com/';
-      const displayUrlText = '/(example.com)';
+      const displayUrlText = 'https://example.com';
      const el = renderer.render({
        type: 'url',
        value: urlText,
--- a/lighthouse-core/test/results/sample_v2.json
+++ b/lighthouse-core/test/results/sample_v2.json
@ -1066,6 +1066,27 @@
        }
      }
    },
+    "uses-rel-preconnect": {
+      "score": 1,
+      "displayValue": "0 ms",
+      "rawValue": 0,
+      "extendedInfo": {
+        "value": []
+      },
+      "scoreDisplayMode": "numeric",
+      "informative": true,
+      "name": "uses-rel-preconnect",
+      "description": "Avoid multiple, costly round trips to any origin",
+      "helpText": "Consider adding preconnect or dns-prefetch resource hints to establish early connections to important third-party origins. [Learn more](https://developers.google.com/web/fundamentals/performance/resource-prioritization#preconnect).",
+      "details": {
+        "type": "table",
+        "headings": [],
+        "items": [],
+        "summary": {
+          "wastedMs": 0
+        }
+      }
+    },
    "font-display": {
      "score": 1,
      "displayValue": "",
@ -4659,6 +4680,11 @@
          "weight": 0,
          "group": "perf-hint"
        },
+        {
+          "id": "uses-rel-preconnect",
+          "weight": 0,
+          "group": "perf-hint"
+        },
        {
          "id": "time-to-first-byte",
          "weight": 0,
--- a/typings/artifacts.d.ts
+++ b/typings/artifacts.d.ts
@ -95,6 +95,8 @@ declare global {

      // TODO(bckenny): remove this for real computed artifacts approach
      requestTraceOfTab(trace: Trace): Promise<Artifacts.TraceOfTab>
+      requestNetworkRecords(devtoolsLogs: DevtoolsLog): Promise<WebInspector.NetworkRequest[]>
+      requestMainResource(devtoolsLogs: DevtoolsLog): Promise<WebInspector.NetworkRequest>
    }

    module Artifacts {
--- a/typings/audit.d.ts
+++ b/typings/audit.d.ts
@ -57,7 +57,7 @@ declare global {
    export interface DetailsRendererDetailsJSON {
      type: 'table';
      headings: Array<Audit.Heading>;
-      items: Array<{[x: string]: string}>;
+      items: Array<{[x: string]: string|number}>;
      summary: DetailsRendererDetailsSummary;
    }

@ -67,7 +67,7 @@ declare global {
      displayValue?: string;
      debugString?: string;
      score?: number;
-      extendedInfo?: {value: string};
+      extendedInfo?: {value: any};
      notApplicable?: boolean;
      error?: boolean;
      // TODO: define details
@ -82,7 +82,7 @@ declare global {
      score: number;
      scoreDisplayMode: ScoringModeValue;
      description: string;
-      extendedInfo?: {value: string};
+      extendedInfo?: {value: any};
      notApplicable?: boolean;
      error?: boolean;
      name: string;
--- a/typings/web-inspector.d.ts
+++ b/typings/web-inspector.d.ts
@ -38,6 +38,7 @@ declare global {
      localizedFailDescription?: string;

      _initiator: NetworkRequestInitiator;
+      initiatorRequest(): NetworkRequest,
      _timing: NetworkRequestTiming;
      _resourceType: ResourceType;
      _mimeType: string;
@ -57,6 +58,8 @@ declare global {
    }

    export interface NetworkRequestTiming {
+      dnsStart: number;
+      dnsEnd: number;
      connectStart: number;
      connectEnd: number;
      sslStart: number;