perf(gather-runner): Clear cache selectively per-pass (#2156)

* Don't _disable_ disk cache, just clear it once for perf.
* Move cache clearing to be per-pass and conditional

lighthouse-core/gather/driver.js

@@ -795,19 +795,12 @@ class Driver {
         .then(_ => this.online = true);
   }
 
-  cleanAndDisableBrowserCaches() {
-    return Promise.all([
-      this.clearBrowserCache(),
-      this.disableBrowserCache()
-    ]);
-  }
-
-  clearBrowserCache() {
-    return this.sendCommand('Network.clearBrowserCache');
-  }
-
-  disableBrowserCache() {
-    return this.sendCommand('Network.setCacheDisabled', {cacheDisabled: true});
+  cleanBrowserCaches() {
+    // Wipe entire disk cache
+    return this.sendCommand('Network.clearBrowserCache')
+      // Toggle 'Disable Cache' to evict the memory cache
+      .then(_ => this.sendCommand('Network.setCacheDisabled', {cacheDisabled: true}))
+      .then(_ => this.sendCommand('Network.setCacheDisabled', {cacheDisabled: false}));
   }
 
   clearDataForOrigin(url) {

lighthouse-core/gather/gather-runner.js

@@ -38,7 +38,7 @@ let GathererResults; // eslint-disable-line no-unused-vars
  *     ii. beginEmulation
  *     iii. enableRuntimeEvents
  *     iv. evaluateScriptOnLoad rescue native Promise from potential polyfill
- *     v. cleanAndDisableBrowserCaches
+ *     v. cleanBrowserCaches
  *     vi. clearDataForOrigin
  *
  * 2. For each pass in the config:
@@ -47,10 +47,12 @@ let GathererResults; // eslint-disable-line no-unused-vars
  *     ii. Enable network request blocking for specified patterns
  *     iii. all gatherers' beforePass()
  *   B. GatherRunner.pass()
- *     i. beginTrace (if requested) & beginDevtoolsLog
- *     ii. GatherRunner.loadPage()
+ *     i. cleanBrowserCaches() (if it's a perf run)
+ *     ii. beginDevtoolsLog()
+ *     iii. beginTrace (if requested)
+ *     iv. GatherRunner.loadPage()
  *       a. navigate to options.url (and wait for onload)
- *     iii. all gatherers' pass()
+ *     v. all gatherers' pass()
  *   C. GatherRunner.afterPass()
  *     i. endTrace (if requested) & endDevtoolsLog & endThrottling
  *     ii. all gatherers' afterPass()
@@ -110,7 +112,6 @@ class GatherRunner {
       .then(_ => driver.enableRuntimeEvents())
       .then(_ => driver.cacheNatives())
       .then(_ => driver.dismissJavaScriptDialogs())
-      .then(_ => resetStorage && driver.cleanAndDisableBrowserCaches())
       .then(_ => resetStorage && driver.clearDataForOrigin(options.url))
       .then(_ => gathererResults.UserAgent = [driver.getUserAgent()]);
   }
@@ -201,16 +202,20 @@ class GatherRunner {
     const config = options.config;
     const gatherers = config.gatherers;
 
+    const recordTrace = config.recordTrace;
+    const isPerfRun = !options.flags.disableStorageReset && recordTrace && config.useThrottling;
+
     const gatherernames = gatherers.map(g => g.name).join(', ');
     const status = 'Loading page & waiting for onload';
     log.log('status', status, gatherernames);
 
-    // Always record devtoolsLog.
-    driver.beginDevtoolsLog();
-
     const pass = Promise.resolve()
-      // Begin tracing only if requested by config.
-      .then(_ => config.recordTrace && driver.beginTrace(options.flags))
+      // Clear disk & memory cache if it's a perf run
+      .then(_ => isPerfRun && driver.cleanBrowserCaches())
+      // Always record devtoolsLog
+      .then(_ => driver.beginDevtoolsLog())
+      // Begin tracing if requested by config.
+      .then(_ => recordTrace && driver.beginTrace(options.flags))
       // Navigate.
       .then(_ => GatherRunner.loadPage(driver, options))
       .then(_ => log.log('statusEnd', status));

lighthouse-core/test/gather/fake-driver.js

@@ -50,7 +50,7 @@ module.exports = {
   evaluateScriptOnLoad() {
     return Promise.resolve();
   },
-  cleanAndDisableBrowserCaches() {},
+  cleanBrowserCaches() {},
   clearDataForOrigin() {},
   cacheNatives() {
     return Promise.resolve();

lighthouse-core/test/gather/gather-runner-test.js

@@ -58,7 +58,7 @@ function getMockedEmulationDriver(emulationFn, netThrottleFn, cpuThrottleFn, blo
     cacheNatives() {
       return Promise.resolve();
     }
-    cleanAndDisableBrowserCaches() {}
+    cleanBrowserCaches() {}
     clearDataForOrigin() {}
     getUserAgent() {
       return Promise.resolve('Fake user agent');
@@ -242,10 +242,10 @@ describe('GatherRunner', function() {
     });
   });
 
-  it('clears the network cache and origin storage', () => {
+  it('clears origin storage', () => {
     const asyncFunc = () => Promise.resolve();
     const tests = {
-      calledDisableNetworkCache: false,
+      calledCleanBrowserCaches: false,
       calledClearStorage: false,
     };
     const createCheck = variable => () => {
@@ -259,22 +259,50 @@ describe('GatherRunner', function() {
       dismissJavaScriptDialogs: asyncFunc,
       enableRuntimeEvents: asyncFunc,
       cacheNatives: asyncFunc,
-      cleanAndDisableBrowserCaches: createCheck('calledDisableNetworkCache'),
+      cleanBrowserCaches: createCheck('calledCleanBrowserCaches'),
       clearDataForOrigin: createCheck('calledClearStorage'),
       blockUrlPatterns: asyncFunc,
       getUserAgent: asyncFunc,
     };
 
     return GatherRunner.setupDriver(driver, {}, {flags: {}}).then(_ => {
-      assert.equal(tests.calledDisableNetworkCache, true);
+      assert.equal(tests.calledCleanBrowserCaches, false);
       assert.equal(tests.calledClearStorage, true);
     });
   });
 
-  it('does not clear the cache & storage when disable-storage-reset flag is set', () => {
+  it('clears the disk & memory cache on a perf run', () => {
     const asyncFunc = () => Promise.resolve();
     const tests = {
-      calledDisableNetworkCache: false,
+      calledCleanBrowserCaches: false
+    };
+    const createCheck = variable => () => {
+      tests[variable] = true;
+      return Promise.resolve();
+    };
+    const driver = {
+      beginDevtoolsLog: asyncFunc,
+      beginTrace: asyncFunc,
+      gotoURL: asyncFunc,
+      cleanBrowserCaches: createCheck('calledCleanBrowserCaches')
+    };
+    const config = {
+      recordTrace: true,
+      useThrottling: true,
+      gatherers: []
+    };
+    const flags = {
+      disableStorageReset: false
+    };
+    return GatherRunner.pass({driver, config, flags}, {TestGatherer: []}).then(_ => {
+      assert.equal(tests.calledCleanBrowserCaches, true);
+    });
+  });
+
+  it('does not clear origin storage with flag --disable-storage-reset', () => {
+    const asyncFunc = () => Promise.resolve();
+    const tests = {
+      calledCleanBrowserCaches: false,
       calledClearStorage: false,
     };
     const createCheck = variable => () => {
@@ -288,7 +316,7 @@ describe('GatherRunner', function() {
       dismissJavaScriptDialogs: asyncFunc,
       enableRuntimeEvents: asyncFunc,
       cacheNatives: asyncFunc,
-      cleanAndDisableBrowserCaches: createCheck('calledDisableNetworkCache'),
+      cleanBrowserCaches: createCheck('calledCleanBrowserCaches'),
       clearDataForOrigin: createCheck('calledClearStorage'),
       blockUrlPatterns: asyncFunc,
       getUserAgent: asyncFunc,
@@ -297,7 +325,7 @@ describe('GatherRunner', function() {
     return GatherRunner.setupDriver(driver, {}, {
       flags: {disableStorageReset: true}
     }).then(_ => {
-      assert.equal(tests.calledDisableNetworkCache, false);
+      assert.equal(tests.calledCleanBrowserCaches, false);
       assert.equal(tests.calledClearStorage, false);
     });
   });
@@ -357,8 +385,9 @@ describe('GatherRunner', function() {
         new TestGatherer()
       ]
     };
+    const flags = {};
 
-    return GatherRunner.pass({driver, config}, {TestGatherer: []}).then(_ => {
+    return GatherRunner.pass({driver, config, flags}, {TestGatherer: []}).then(_ => {
       assert.equal(calledTrace, true);
     });
   });
@@ -405,8 +434,9 @@ describe('GatherRunner', function() {
         new TestGatherer()
       ]
     };
+    const flags = {};
 
-    return GatherRunner.pass({driver, config}, {TestGatherer: []}).then(_ => {
+    return GatherRunner.pass({driver, config, flags}, {TestGatherer: []}).then(_ => {
       assert.equal(calledDevtoolsLogCollect, true);
     });
   });