Bug 1299118: Measure TTI (or TTFI at the moment) in Raptor TP6 r=rwood

modules/libpref/init/all.js

@@ -182,8 +182,8 @@ pref("dom.performance.time_to_non_blank_paint.enabled", false);
 // Enable exposing timeToDOMContentFlushed
 pref("dom.performance.time_to_dom_content_flushed.enabled", false);
 
-// Enable exposing timeToInteractive
-pref("dom.performance.time_to_interactive.enabled", false);
+// Enable exposing timeToFirstInteractive
+pref("dom.performance.time_to_first_interactive.enabled", false);
 
 // Enable requestIdleCallback API
 pref("dom.requestIdleCallback.enabled", true);

taskcluster/ci/test/raptor.yml

@@ -37,7 +37,7 @@ raptor-tp6-firefox:
     try-name: raptor-tp6-firefox
     treeherder-symbol: Rap(tp6)
     run-on-projects: ['try', 'mozilla-central']
-    max-run-time: 1200
+    max-run-time: 1800
     mozharness:
         extra-options:
             - --test=raptor-tp6

testing/profiles/raptor/user.js

@@ -2,6 +2,7 @@
 /* globals user_pref */
 user_pref("dom.performance.time_to_non_blank_paint.enabled", true);
 user_pref("dom.performance.time_to_dom_content_flushed.enabled", true);
+user_pref("dom.performance.time_to_first_interactive.enabled", true);
 
 // required for geckoview logging
 user_pref("geckoview.console.enabled", true);

testing/raptor/raptor/manifest.py

@@ -84,6 +84,8 @@ def write_test_settings_json(test_details, oskey):
             test_settings['raptor-options']['measure']['fcp'] = True
         if "hero" in test_details['measure']:
             test_settings['raptor-options']['measure']['hero'] = test_details['hero'].split()
+        if "ttfi" in test_details['measure']:
+            test_settings['raptor-options']['measure']['ttfi'] = True
     if test_details.get("page_timeout", None) is not None:
         test_settings['raptor-options']['page_timeout'] = int(test_details['page_timeout'])
     test_settings['raptor-options']['unit'] = test_details.get("unit", "ms")

testing/raptor/raptor/tests/raptor-tp6.ini

@@ -14,19 +14,22 @@ page_cycles = 25
 unit = ms
 lower_is_better = true
 alert_threshold = 2.0
+# TTI/TTFI can take a while on some pages, and requires at least 5 seconds
+# beyond typical pageload time
+page_timeout = 30000
 
 [raptor-tp6-amazon-firefox]
 apps = firefox
 test_url = https://www.amazon.com/s/url=search-alias%3Daps&field-keywords=laptop
 playback_recordings = amazon.mp
-measure = fnbpaint, hero, dcf
+measure = fnbpaint, hero, dcf, ttfi
 hero = hero1
 
 [raptor-tp6-facebook-firefox]
 apps = firefox
 test_url = https://www.facebook.com
 playback_recordings = facebook.mp
-measure = fnbpaint, hero, dcf
+measure = fnbpaint, hero, dcf, ttfi
 hero = hero1
 
 [raptor-tp6-google-firefox]
@@ -36,7 +39,7 @@ apps = firefox
 # to be loaded into that page also; resulting in 2 fnbpaint values etc.
 test_url = https://www.google.com/search?hl=en&q=barack+obama&cad=h
 playback_recordings = google-search.mp
-measure = fnbpaint, hero, dcf
+measure = fnbpaint, hero, dcf, ttfi
 hero = hero1
 
 [raptor-tp6-youtube-firefox]

testing/raptor/webext/raptor/measure.js

@@ -23,6 +23,12 @@ var getFNBPaint = false;
 // default only; this is set via control server settings json
 var getDCF = false;
 
+// measure firefox TTFI
+// note: this browser pref must be enabled:
+// dom.performance.time_to_first_interactive.enabled = True
+// default only; this is set via control server settings json
+var getTTFI = false;
+
 // measure google's first-contentful-paint
 // default only; this is set via control server settings json
 var getFCP = false;
@@ -87,6 +93,14 @@ function setup(settings) {
       measureHero();
     }
   }
+
+  if (settings.measure.ttfi !== undefined) {
+    getTTFI = settings.measure.ttfi;
+    if (getTTFI) {
+      console.log("will be measuring ttfi");
+      measureTTFI();
+    }
+  }
 }
 
 function measureHero() {
@@ -179,6 +193,37 @@ function measureDCF() {
   }
 }
 
+function measureTTFI() {
+  var x = window.performance.timing.timeToFirstInteractive;
+
+  if (typeof(x) == "undefined") {
+    console.log("ERROR: timeToFirstInteractive is undefined; ensure the pref is enabled");
+    return;
+  }
+  if (x > 0) {
+    console.log("got timeToFirstInteractive: " + x);
+    gRetryCounter = 0;
+    var startTime = perfData.timing.fetchStart;
+    sendResult("ttfi", x - startTime);
+  } else {
+    gRetryCounter += 1;
+    // NOTE: currently the gecko implementation doesn't look at network
+    // requests, so this is closer to TimeToFirstInteractive than
+    // TimeToInteractive.  Also, we use FNBP instead of FCP as the start
+    // point.  TTFI/TTI requires running at least 5 seconds past last
+    // "busy" point, give 25 seconds here (overall the harness times out at
+    // 30 seconds).  Some pages will never get 5 seconds without a busy
+    // period!
+    if (gRetryCounter <= 25*(1000/200)) {
+      console.log("\TTFI is not yet available (0), retry number " + gRetryCounter + "...\n");
+      window.setTimeout(measureTTFI, 200);
+    } else {
+      // unable to get a value for TTFI - filter out later
+      sendResult("ttfi", 0);
+    }
+  }
+}
+
 function measureFirstContentfulPaint() {
   // see https://developer.mozilla.org/en-US/docs/Web/API/PerformancePaintTiming
   var resultType = "fcp";

testing/raptor/webext/raptor/runner.js

@@ -37,12 +37,14 @@ var getHero = false;
 var getFNBPaint = false;
 var getFCP = false;
 var getDCF = false;
+var getTTFI = false;
 var isHeroPending = false;
 var pendingHeroes = [];
 var settings = {};
 var isFNBPaintPending = false;
 var isFCPPending = false;
 var isDCFPending = false;
+var isTTFIPending = false;
 var isBenchmarkPending = false;
 var pageTimeout = 10000; // default pageload timeout
 
@@ -106,6 +108,9 @@ function getTestSettings() {
                 getHero = true;
               }
             }
+            if (settings.measure.ttfi !== undefined) {
+              getTTFI = settings.measure.ttfi;
+            }
           } else {
             console.log("abort: 'measure' key not found in test settings");
             cleanUp();
@@ -177,7 +182,7 @@ function waitForResult() {
   return new Promise(resolve => {
     function checkForResult() {
       if (testType == "pageload") {
-        if (!isHeroPending && !isFNBPaintPending && !isFCPPending && !isDCFPending) {
+        if (!isHeroPending && !isFNBPaintPending && !isFCPPending && !isDCFPending && !isTTFIPending) {
           cancelTimeoutAlarm("raptor-page-timeout");
           resolve();
         } else {
@@ -222,6 +227,8 @@ function nextCycle() {
           isFCPPending = true;
         if (getDCF)
           isDCFPending = true;
+        if (getTTFI)
+          isTTFIPending = true;
       } else if (testType == "benchmark") {
         isBenchmarkPending = true;
       }
@@ -298,6 +305,9 @@ function resultListener(request, sender, sendResponse) {
       } else if (request.type == "dcf") {
         results.measurements.dcf.push(request.value);
         isDCFPending = false;
+      } else if (request.type == "ttfi") {
+        results.measurements.ttfi.push(request.value);
+        isTTFIPending = false;
       } else if (request.type == "fcp") {
         results.measurements.fcp.push(request.value);
         isFCPPending = false;