core(network-analyzer): infer RTT from receiveHeadersEnd (#5694)

lighthouse-core/lib/dependency-graph/simulator/network-analyzer.js

@@ -8,6 +8,17 @@
 const INITIAL_CWD = 14 * 1024;
 const NetworkRequest = require('../../network-request');
 
+// Assume that 40% of TTFB was server response time by default for static assets
+const DEFAULT_SERVER_RESPONSE_PERCENTAGE = 0.4;
+
+// For certain resource types, server response time takes up a greater percentage of TTFB (dynamic
+// assets like HTML documents, XHR/API calls, etc)
+const SERVER_RESPONSE_PERCENTAGE_OF_TTFB = {
+  Document: 0.9,
+  XHR: 0.9,
+  Fetch: 0.9,
+};
+
 class NetworkAnalyzer {
   /**
    * @return {string}
@@ -149,7 +160,7 @@ class NetworkAnalyzer {
    * Estimates the observed RTT to each origin based on how long it took until Chrome could
    * start sending the actual request when a new connection was required.
    * NOTE: this will tend to overestimate the actual RTT as the request can be delayed for other
-   * reasons as well such as DNS lookup.
+   * reasons as well such as more SSL handshakes if TLS False Start is not enabled.
    *
    * @param {LH.Artifacts.NetworkRequest[]} records
    * @return {Map<string, number[]>}
@@ -159,14 +170,48 @@ class NetworkAnalyzer {
       if (connectionReused) return;
       if (!Number.isFinite(timing.sendStart) || timing.sendStart < 0) return;
 
-      // Assume everything before sendStart was just a TCP handshake
-      // 1 RT needed for http, 2 RTs for https
-      let roundTrips = 1;
+      // Assume everything before sendStart was just DNS + (SSL)? + TCP handshake
+      // 1 RT for DNS, 1 RT (maybe) for SSL, 1 RT for TCP
+      let roundTrips = 2;
       if (record.parsedURL.scheme === 'https') roundTrips += 1;
       return timing.sendStart / roundTrips;
     });
   }
 
+  /**
+   * Estimates the observed RTT to each origin based on how long it took until Chrome received the
+   * headers of the response (~TTFB).
+   * NOTE: this is the most inaccurate way to estimate the RTT, but in some environments it's all
+   * we have access to :(
+   *
+   * @param {LH.Artifacts.NetworkRequest[]} records
+   * @return {Map<string, number[]>}
+   */
+  static _estimateRTTByOriginViaHeadersEndTiming(records) {
+    return NetworkAnalyzer._estimateValueByOrigin(records, ({record, timing, connectionReused}) => {
+      if (!Number.isFinite(timing.receiveHeadersEnd) || timing.receiveHeadersEnd < 0) return;
+
+      const serverResponseTimePercentage = SERVER_RESPONSE_PERCENTAGE_OF_TTFB[record.resourceType]
+        || DEFAULT_SERVER_RESPONSE_PERCENTAGE;
+      const estimatedServerResponseTime = timing.receiveHeadersEnd * serverResponseTimePercentage;
+
+      // When connection was reused...
+      // TTFB = 1 RT for request + server response time
+      let roundTrips = 1;
+
+      // When connection was fresh...
+      // TTFB = DNS + (SSL)? + TCP handshake + 1 RT for request + server response time
+      if (!connectionReused) {
+        roundTrips += 1; // DNS
+        if (record.parsedURL.scheme === 'https') roundTrips += 1; // SSL
+        roundTrips += 1; // TCP handshake
+      }
+
+      // subtract out our estimated server response time
+      return Math.max((timing.receiveHeadersEnd - estimatedServerResponseTime) / roundTrips, 3);
+    });
+  }
+
   /**
    * Given the RTT to each origin, estimates the observed server response times.
    *
@@ -264,7 +309,11 @@ class NetworkAnalyzer {
         forceCoarseEstimates: false,
         // coarse estimates include lots of extra time and noise
         // multiply by some factor to deflate the estimates a bit
-        coarseEstimateMultiplier: 0.5,
+        coarseEstimateMultiplier: 0.3,
+        // useful for testing to isolate the different methods of estimation
+        useDownloadEstimates: true,
+        useSendStartEstimates: true,
+        useHeadersEndEstimates: true,
       },
       options
     );
@@ -274,12 +323,21 @@ class NetworkAnalyzer {
       estimatesByOrigin = new Map();
       const estimatesViaDownload = NetworkAnalyzer._estimateRTTByOriginViaDownloadTiming(records);
       const estimatesViaSendStart = NetworkAnalyzer._estimateRTTByOriginViaSendStartTiming(records);
+      const estimatesViaTTFB = NetworkAnalyzer._estimateRTTByOriginViaHeadersEndTiming(records);
 
       for (const [origin, estimates] of estimatesViaDownload.entries()) {
+        if (!options.useDownloadEstimates) continue;
         estimatesByOrigin.set(origin, estimates);
       }
 
       for (const [origin, estimates] of estimatesViaSendStart.entries()) {
+        if (!options.useSendStartEstimates) continue;
+        const existing = estimatesByOrigin.get(origin) || [];
+        estimatesByOrigin.set(origin, existing.concat(estimates));
+      }
+
+      for (const [origin, estimates] of estimatesViaTTFB.entries()) {
+        if (!options.useHeadersEndEstimates) continue;
         const existing = estimatesByOrigin.get(origin) || [];
         estimatesByOrigin.set(origin, existing.concat(estimates));
       }

lighthouse-core/test/lib/dependency-graph/simulator/network-analyzer-test.js

@@ -144,10 +144,10 @@ describe('DependencyGraph/Simulator/NetworkAnalyzer', () => {
     });
 
     it('should infer from sendStart when available', () => {
-      const timing = {sendStart: 100};
-      // this record took 100ms before Chrome could send the request
+      const timing = {sendStart: 150};
+      // this record took 150ms before Chrome could send the request
       // i.e. DNS (maybe) + queuing (maybe) + TCP handshake took ~100ms
-      // 100ms / 2 round trips ~= 50ms RTT
+      // 150ms / 3 round trips ~= 50ms RTT
       const record = createRecord({startTime: 0, endTime: 1, timing});
       const result = NetworkAnalyzer.estimateRTTByOrigin([record], {coarseEstimateMultiplier: 1});
       const expected = {min: 50, max: 50, avg: 50, median: 50};
@@ -160,13 +160,31 @@ describe('DependencyGraph/Simulator/NetworkAnalyzer', () => {
       // i.e. it took at least one full additional roundtrip after first byte to download the rest
       // 1000ms / 1 round trip ~= 1000ms RTT
       const record = createRecord({startTime: 0, endTime: 1.1, transferSize: 28 * 1024, timing});
-      const result = NetworkAnalyzer.estimateRTTByOrigin([record], {coarseEstimateMultiplier: 1});
+      const result = NetworkAnalyzer.estimateRTTByOrigin([record], {
+        coarseEstimateMultiplier: 1,
+        useHeadersEndEstimates: false,
+      });
       const expected = {min: 1000, max: 1000, avg: 1000, median: 1000};
       assert.deepStrictEqual(result.get('https://example.com'), expected);
     });
 
+    it('should infer from TTFB when available', () => {
+      const timing = {receiveHeadersEnd: 1000};
+      const record = createRecord({startTime: 0, endTime: 1, timing});
+      const result = NetworkAnalyzer.estimateRTTByOrigin([record], {
+        coarseEstimateMultiplier: 1,
+      });
+
+      // this record's TTFB was 1000ms, it used SSL and was a fresh connection requiring a handshake
+      // which needs ~4 RTs. We don't know its resource type so it'll be assumed that 40% of it was
+      // server response time.
+      // 600 ms / 4 = 150ms
+      const expected = {min: 150, max: 150, avg: 150, median: 150};
+      assert.deepStrictEqual(result.get('https://example.com'), expected);
+    });
+
     it('should handle untrustworthy connection information', () => {
-      const timing = {sendStart: 100};
+      const timing = {sendStart: 150};
       const recordA = createRecord({startTime: 0, endTime: 1, timing, connectionReused: true});
       const recordB = createRecord({
         startTime: 0,