Bug 1622088 - Part 2: Add some mochitest-browser tests for the about:home startup cache. r=k88hudson

Differential Revision: https://phabricator.services.mozilla.com/D71793

browser/components/BrowserGlue.jsm

@@ -5005,6 +5005,10 @@ var AboutHomeStartupCache = {
     Services.obs.addObserver(this, "process-type-set");
     Services.obs.addObserver(this, "ipc:content-shutdown");
 
+    this._cacheEntryPromise = new Promise(resolve => {
+      this._cacheEntryResolver = resolve;
+    });
+
     let lci = Services.loadContextInfo.default;
     let storage = Services.cache2.diskCacheStorage(lci, false);
     try {
@@ -5054,6 +5058,9 @@ var AboutHomeStartupCache = {
     this._initted = false;
     this._cacheEntry = null;
     this._hasWrittenThisSession = false;
+    this._cacheEntryPromise = null;
+    this._cacheEntryResolver = null;
+
     this.log.trace("Uninitialized.");
     this.log.removeAppender(this._appender);
     this.log = null;
@@ -5111,7 +5118,7 @@ var AboutHomeStartupCache = {
    *   Resolves when a fresh version of the cache has been written.
    */
   async cacheNow() {
-    this._hasWrittenThisSession = true;
+    this.log.trace("Caching now.");
     this._cacheProgress = "Getting cache streams";
     let { pageInputStream, scriptInputStream } = await this.requestCache();
 
@@ -5123,6 +5130,8 @@ var AboutHomeStartupCache = {
     this._cacheProgress = "Writing to cache";
     await this.populateCache(pageInputStream, scriptInputStream);
     this._cacheProgress = "Done";
+    this.log.trace("Done writing to cache.");
+    this._hasWrittenThisSession = true;
   },
 
   /**
@@ -5240,7 +5249,7 @@ var AboutHomeStartupCache = {
     if (parseInt(version, 10) != this.CACHE_VERSION) {
       this.log.info("Version does not match! Dooming and closing streams.\n");
       // This cache is no good - doom it, and prepare for a new one.
-      this._cacheEntry = this._cacheEntry.recreate();
+      this.clearCache();
       this.pagePipe.outputStream.close();
       this.scriptPipe.outputStream.close();
       return;
@@ -5341,40 +5350,120 @@ var AboutHomeStartupCache = {
    *   A stream containing the HTML markup to be saved to the cache.
    * @param scriptInputStream (nsIInputStream)
    *   A stream containing the JS hydration script to be saved to the cache.
+   * @returns Promise
+   * @resolves undefined
+   *   When the cache has been successfully written to.
+   * @rejects Error
+   *   Rejects with a JS Error if writing any part of the cache happens to
+   *   fail.
    */
-  populateCache(pageInputStream, scriptInputStream) {
-    // Doom the old cache entry, so we can start writing to a new one.
-    this.log.trace("Populating the cache. Dooming old entry.");
-    this._cacheEntry = this._cacheEntry.recreate();
+  async populateCache(pageInputStream, scriptInputStream) {
+    await this.ensureCacheEntry();
 
-    this.log.trace("Opening the page output stream.");
-    let pageOutputStream = this._cacheEntry.openOutputStream(0, -1);
+    await new Promise((resolve, reject) => {
+      // Doom the old cache entry, so we can start writing to a new one.
+      this.log.trace("Populating the cache. Dooming old entry.");
+      this.clearCache();
 
-    this.log.info("Writing the page cache.");
-    NetUtil.asyncCopy(pageInputStream, pageOutputStream, () => {
-      this.log.trace(
-        "Writing the page data is complete. Now opening the " +
-          "script output stream."
-      );
+      this.log.trace("Opening the page output stream.");
+      let pageOutputStream;
+      try {
+        pageOutputStream = this._cacheEntry.openOutputStream(0, -1);
+      } catch (e) {
+        reject(e);
+        return;
+      }
 
-      let scriptOutputStream = this._cacheEntry.openAlternativeOutputStream(
-        "script",
-        -1
-      );
+      this.log.info("Writing the page cache.");
+      NetUtil.asyncCopy(pageInputStream, pageOutputStream, pageResult => {
+        if (!Components.isSuccessCode(pageResult)) {
+          this.log.error("Failed to write page. Result: " + pageResult);
+          reject(new Error(pageResult));
+          return;
+        }
 
-      this.log.info("Writing the script cache.");
-      NetUtil.asyncCopy(scriptInputStream, scriptOutputStream, () => {
-        this.log.trace("Writing the script cache is done. Setting version.");
-        this._cacheEntry.setMetaDataElement(
-          "version",
-          String(this.CACHE_VERSION)
+        this.log.trace(
+          "Writing the page data is complete. Now opening the " +
+            "script output stream."
+        );
+
+        let scriptOutputStream;
+        try {
+          scriptOutputStream = this._cacheEntry.openAlternativeOutputStream(
+            "script",
+            -1
+          );
+        } catch (e) {
+          reject(e);
+          return;
+        }
+
+        this.log.info("Writing the script cache.");
+        NetUtil.asyncCopy(
+          scriptInputStream,
+          scriptOutputStream,
+          scriptResult => {
+            if (!Components.isSuccessCode(scriptResult)) {
+              this.log.error("Failed to write script. Result: " + scriptResult);
+              reject(new Error(scriptResult));
+              return;
+            }
+
+            this.log.trace(
+              "Writing the script cache is done. Setting version."
+            );
+            try {
+              this._cacheEntry.setMetaDataElement(
+                "version",
+                String(this.CACHE_VERSION)
+              );
+            } catch (e) {
+              this.log.error("Failed to write version.");
+              reject(e);
+              return;
+            }
+            this.log.trace(`Version is set to ${this.CACHE_VERSION}.`);
+            this.log.info("Caching of page and script is done.");
+            resolve();
+          }
         );
-        this.log.trace(`Version is set to ${this.CACHE_VERSION}.`);
-        this.log.info("Caching of page and script is done.");
       });
     });
   },
 
+  /**
+   * Returns a Promise that resolves once the nsICacheEntry for the cache
+   * is available to write to and read from.
+   *
+   * @returns Promise
+   * @resolves nsICacheEntry
+   *   Once the cache entry has become available.
+   * @rejects String
+   *   Rejects with an error message if getting the cache entry is attempted
+   *   before the AboutHomeStartupCache component has been initialized.
+   */
+  ensureCacheEntry() {
+    if (!this._initted) {
+      return Promise.reject(
+        "Cannot ensureCacheEntry - AboutHomeStartupCache is not initted"
+      );
+    }
+
+    return this._cacheEntryPromise;
+  },
+
+  /**
+   * Clears the contents of the cache.
+   */
+  clearCache() {
+    this.log.trace("Clearing the cache.");
+    this._cacheEntry = this._cacheEntry.recreate();
+    this._cacheEntryPromise = new Promise(resolve => {
+      resolve(this._cacheEntry);
+    });
+    this._hasWrittenThisSession = false;
+  },
+
   /**
    * Called when a content process is created. If this is the "privileged
    * about content process", then the cache streams will be sent to it.
@@ -5497,5 +5586,7 @@ var AboutHomeStartupCache = {
     this._cacheEntry = aEntry;
     this.makePipes();
     this.maybeConnectToPipes();
+
+    this._cacheEntryResolver(this._cacheEntry);
   },
 };

browser/components/newtab/AboutNewTabService.jsm

@@ -52,6 +52,8 @@ const { E10SUtils } = ChromeUtils.import(
 
 const PREF_ABOUT_HOME_CACHE_ENABLED =
   "browser.startup.homepage.abouthome_cache.enabled";
+const PREF_ABOUT_HOME_CACHE_TESTING =
+  "browser.startup.homepage.abouthome_cache.testing";
 const PREF_SEPARATE_ABOUT_WELCOME = "browser.aboutwelcome.enabled";
 const SEPARATE_ABOUT_WELCOME_URL =
   "resource://activity-stream/aboutwelcome/aboutwelcome.html";
@@ -120,6 +122,24 @@ const AboutHomeStartupCacheChild = {
     this._initted = true;
   },
 
+  /**
+   * A function that lets us put the AboutHomeStartupCacheChild back into
+   * its initial state. This is used by tests to let us simulate the startup
+   * behaviour of the module without having to manually launch a new privileged
+   * about content process every time.
+   */
+  uninit() {
+    if (!Services.prefs.getBoolPref(PREF_ABOUT_HOME_CACHE_TESTING, false)) {
+      throw new Error(
+        "Cannot uninit AboutHomeStartupCacheChild unless testing."
+      );
+    }
+
+    this._pageInputStream = null;
+    this._scriptInputStream = null;
+    this._initted = false;
+  },
+
   /**
    * A public method called from nsIAboutNewTabService that attempts
    * return an nsIChannel for a cached about:home document that we

browser/components/newtab/moz.build

@@ -7,7 +7,10 @@
 with Files("**"):
     BUG_COMPONENT = ("Firefox", "New Tab Page")
 
-BROWSER_CHROME_MANIFESTS += ['test/browser/browser.ini']
+BROWSER_CHROME_MANIFESTS += [
+  'test/browser/abouthomecache/browser.ini',
+  'test/browser/browser.ini',
+]
 
 SPHINX_TREES['docs'] = 'docs'
 SPHINX_TREES['content-src/asrouter/docs'] = 'content-src/asrouter/docs'

browser/components/newtab/test/browser/abouthomecache/browser.ini

@@ -0,0 +1,20 @@
+[DEFAULT]
+support-files =
+  head.js
+prefs =
+  browser.tabs.remote.separatePrivilegedContentProcess=true
+  browser.startup.homepage.abouthome_cache.enabled=true
+  browser.startup.homepage.abouthome_cache.cache_on_shutdown=false
+  browser.startup.homepage.abouthome_cache.testing=true
+  browser.startup.page=1
+  browser.newtabpage.activity-stream.discoverystream.endpoints=data:
+  browser.newtabpage.activity-stream.feeds.section.topstories=true
+  browser.newtabpage.activity-stream.feeds.section.topstories.options={"provider_name":""}
+  browser.newtabpage.activity-stream.telemetry.structuredIngestion=false
+
+[browser_basic_endtoend.js]
+[browser_bump_version.js]
+[browser_no_cache.js]
+[browser_overwrite_cache.js]
+[browser_process_crash.js]
+skip-if = !e10s || !crashreporter

browser/components/newtab/test/browser/abouthomecache/browser_basic_endtoend.js

@@ -0,0 +1,22 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+/**
+ * Tests that the about:home cache gets written on shutdown, and read
+ * from in the subsequent startup.
+ */
+add_task(async function test_basic_behaviour() {
+  await BrowserTestUtils.withNewTab("about:home", async browser => {
+    // First, clear the cache to test the base case.
+    await clearCache();
+    await simulateRestart(browser);
+    await ensureCachedAboutHome(browser);
+
+    // Next, test that a subsequent restart also shows the cached
+    // about:home.
+    await simulateRestart(browser);
+    await ensureCachedAboutHome(browser);
+  });
+});

browser/components/newtab/test/browser/abouthomecache/browser_bump_version.js

@@ -0,0 +1,21 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+/**
+ * Test that if the "version" metadata on the cache entry doesn't match
+ * the expectation that we ignore the cache and load the dynamic about:home
+ * document.
+ */
+add_task(async function test_bump_version() {
+  await BrowserTestUtils.withNewTab("about:home", async browser => {
+    // First, ensure that a pre-existing cache exists.
+    await simulateRestart(browser);
+
+    let cacheEntry = await AboutHomeStartupCache.ensureCacheEntry();
+    cacheEntry.setMetaDataElement("version", "somethingnew");
+    await simulateRestart(browser, false /* withAutoShutdownWrite */);
+    await ensureDynamicAboutHome(browser);
+  });
+});

browser/components/newtab/test/browser/abouthomecache/browser_no_cache.js

@@ -0,0 +1,18 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+requestLongerTimeout(2);
+
+/**
+ * Test that if there's no cache written, that we load the dynamic
+ * about:home document on startup.
+ */
+add_task(async function test_no_cache() {
+  await BrowserTestUtils.withNewTab("about:home", async browser => {
+    await clearCache();
+    await simulateRestart(browser, false /* withAutoShutdownWrite */);
+    await ensureDynamicAboutHome(browser);
+  });
+});

browser/components/newtab/test/browser/abouthomecache/browser_overwrite_cache.js

@@ -0,0 +1,37 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+/**
+ * Tests that if a pre-existing about:home cache exists, that it can
+ * be overwritten with new information.
+ */
+add_task(async function test_overwrite_cache() {
+  await BrowserTestUtils.withNewTab("about:home", async browser => {
+    await simulateRestart(browser);
+    const TEST_ID = "test_overwrite_cache_h1";
+
+    // We need the CSP meta tag in about: pages, otherwise we hit assertions in
+    // debug builds.
+    await injectIntoCache(
+      `
+      <html>
+        <head>
+        <meta http-equiv="Content-Security-Policy" content="default-src 'none'; object-src 'none'; script-src resource: chrome:; connect-src https:; img-src https: data: blob:; style-src 'unsafe-inline';">
+        </head>
+        <body>
+          <h1 id="${TEST_ID}">Something new</h1>
+        </body>
+        <script src="about:home?jscache"></script>
+      </html>`,
+      "window.__FROM_STARTUP_CACHE__ = true;"
+    );
+    await simulateRestart(browser, false /* withAutoShutdownWrite */);
+
+    await SpecialPowers.spawn(browser, [TEST_ID], async testID => {
+      let target = content.document.getElementById(testID);
+      Assert.ok(target, "Found the target element");
+    });
+  });
+});

browser/components/newtab/test/browser/abouthomecache/browser_process_crash.js

@@ -0,0 +1,41 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+/**
+ * Test that if the "privileged about content process" crashes, that it
+ * drops its internal reference to the "privileged about content process"
+ * process manager, and that a subsequent restart of that process type
+ * results in a new cached document load. Also tests that crashing of
+ * any other content process type doesn't clear the process manager
+ * reference.
+ */
+add_task(async function test_process_crash() {
+  await BrowserTestUtils.withNewTab("about:home", async browser => {
+    await simulateRestart(browser);
+    let origProcManager = AboutHomeStartupCache._procManager;
+
+    await BrowserTestUtils.crashFrame(browser);
+    Assert.notEqual(
+      origProcManager,
+      AboutHomeStartupCache._procManager,
+      "Should have dropped the reference to the crashed process"
+    );
+  });
+
+  let latestProcManager = AboutHomeStartupCache._procManager;
+
+  await BrowserTestUtils.withNewTab("about:home", async browser => {
+    await ensureCachedAboutHome(browser);
+  });
+
+  await BrowserTestUtils.withNewTab("http://example.com", async browser => {
+    await BrowserTestUtils.crashFrame(browser);
+    Assert.equal(
+      latestProcManager,
+      AboutHomeStartupCache._procManager,
+      "Should still have the reference to the privileged about process"
+    );
+  });
+});

browser/components/newtab/test/browser/abouthomecache/head.js

@@ -0,0 +1,238 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+let { AboutHomeStartupCache } = ChromeUtils.import(
+  "resource:///modules/BrowserGlue.jsm"
+);
+
+/**
+ * Shuts down the AboutHomeStartupCache components in the parent process
+ * and privileged about content process, and then restarts them, simulating
+ * the parent process having restarted.
+ *
+ * @param browser (<xul:browser>)
+ *   A <xul:browser> with about:home running in it. This will be reloaded
+ *   after the restart simultion is complete, and that reload will attempt
+ *   to read any about:home cache contents.
+ * @param options (object, optional)
+ *
+ *   An object with the following properties:
+ *
+ *     withAutoShutdownWrite (boolean, optional):
+ *       Whether or not the shutdown part of the simulation should cause the
+ *       shutdown handler to run, which normally causes the cache to be
+ *       written. Setting this to false is handy if the cache has been
+ *       specially prepared for the subsequent startup, and we don't want to
+ *       overwrite it. This defaults to true.
+ *
+ *     ensureCacheWinsRace (boolean, optional):
+ *       Ensures that the privileged about content process will be able to
+ *       read the bytes from the streams sent down from the HTTP cache. Use
+ *       this to avoid the HTTP cache "losing the race" against reading the
+ *       about:home document from the omni.ja. This defaults to true.
+ *
+ * @returns Promise
+ * @resolves undefined
+ *   Resolves once the restart simulation is complete, and the <xul:browser>
+ *   pointed at about:home finishes reloading.
+ */
+// eslint-disable-next-line no-unused-vars
+async function simulateRestart(
+  browser,
+  { withAutoShutdownWrite, ensureCacheWinsRace } = {
+    withAutoShutdownWrite: true,
+    ensureCacheWinsRace: true,
+  }
+) {
+  info("Simulating restart of the browser");
+  let processManager = browser.messageManager.processMessageManager;
+  if (processManager.remoteType !== E10SUtils.PRIVILEGEDABOUT_REMOTE_TYPE) {
+    throw new Error(
+      "prepareLoadFromCache should only be called on a browser " +
+        "loaded in the privileged about content process."
+    );
+  }
+
+  if (withAutoShutdownWrite) {
+    info("Simulating shutdown write");
+    await AboutHomeStartupCache.onShutdown();
+    info("Shutdown write done");
+  } else {
+    info("Intentionally skipping shutdown write");
+  }
+
+  AboutHomeStartupCache.uninit();
+
+  info("Waiting for AboutHomeStartupCacheChild to uninit");
+  await SpecialPowers.spawn(browser, [], async () => {
+    let { AboutHomeStartupCacheChild } = ChromeUtils.import(
+      "resource:///modules/AboutNewTabService.jsm"
+    );
+    AboutHomeStartupCacheChild.uninit();
+  });
+  info("AboutHomeStartupCacheChild uninitted");
+
+  AboutHomeStartupCache.init();
+
+  AboutHomeStartupCache.sendCacheInputStreams(processManager);
+
+  info("Waiting for AboutHomeStartupCache cache entry");
+  await AboutHomeStartupCache.ensureCacheEntry();
+  info("Got AboutHomeStartupCache cache entry");
+
+  if (ensureCacheWinsRace) {
+    info("Ensuring cache bytes are available");
+    await SpecialPowers.spawn(browser, [], async () => {
+      let { AboutHomeStartupCacheChild } = ChromeUtils.import(
+        "resource:///modules/AboutNewTabService.jsm"
+      );
+      let pageStream = AboutHomeStartupCacheChild._pageInputStream;
+      let scriptStream = AboutHomeStartupCacheChild._scriptInputStream;
+      await ContentTaskUtils.waitForCondition(() => {
+        return pageStream.available() && scriptStream.available();
+      });
+    });
+  }
+
+  info("Waiting for about:home to load");
+  let loaded = BrowserTestUtils.browserLoaded(browser, false, "about:home");
+  BrowserTestUtils.loadURI(browser, "about:home");
+  await loaded;
+  info("about:home loaded");
+}
+
+/**
+ * Writes a page string and a script string into the cache for
+ * the next about:home load.
+ *
+ * @param page (String)
+ *   The HTML content to write into the cache. This cannot be the empty
+ *   string.
+ * @param script (String)
+ *   The JS content to write into the cache that can be loaded via
+ *   about:home?jscache. This cannot be the empty string.
+ * @returns Promise
+ * @resolves undefined
+ *   When the page and script content has been successfully written.
+ */
+// eslint-disable-next-line no-unused-vars
+async function injectIntoCache(page, script) {
+  if (!page || !script) {
+    throw new Error("Cannot injectIntoCache with falsey values");
+  }
+
+  await AboutHomeStartupCache.ensureCacheEntry();
+
+  let pageInputStream = Cc[
+    "@mozilla.org/io/string-input-stream;1"
+  ].createInstance(Ci.nsIStringInputStream);
+
+  pageInputStream.setUTF8Data(page);
+
+  let scriptInputStream = Cc[
+    "@mozilla.org/io/string-input-stream;1"
+  ].createInstance(Ci.nsIStringInputStream);
+
+  scriptInputStream.setUTF8Data(script);
+
+  await AboutHomeStartupCache.populateCache(pageInputStream, scriptInputStream);
+}
+
+/**
+ * Clears out any pre-existing about:home cache.
+ * @returns Promise
+ * @resolves undefined
+ *   Resolves when the cache is cleared.
+ */
+// eslint-disable-next-line no-unused-vars
+async function clearCache() {
+  info("Test is clearing the cache");
+  AboutHomeStartupCache.clearCache();
+  await AboutHomeStartupCache.ensureCacheEntry();
+  info("Test has cleared the cache.");
+}
+
+/**
+ * Tests that the about:home document loaded in a passed <xul:browser> was
+ * one from the cache.
+ *
+ * We test for this by looking for some tell-tale signs of the cached
+ * document:
+ *
+ *   1. The about:home?jscache <script> element
+ *   2. The __FROM_STARTUP_CACHE__ expando on the window
+ *   3. The "activity-stream" class on the document body
+ *   4. The top sites section
+ *
+ * @returns Promise
+ * @resolves undefined
+ *   Resolves once the cache entry has been destroyed.
+ */
+// eslint-disable-next-line no-unused-vars
+async function ensureCachedAboutHome(browser) {
+  await SpecialPowers.spawn(browser, [], async () => {
+    let scripts = Array.from(content.document.querySelectorAll("script"));
+    Assert.ok(!!scripts.length, "There should be page scripts.");
+    let [lastScript] = scripts.reverse();
+    Assert.equal(
+      lastScript.src,
+      "about:home?jscache",
+      "Found about:home?jscache script tag, indicating the cached doc"
+    );
+    Assert.ok(
+      Cu.waiveXrays(content).__FROM_STARTUP_CACHE__,
+      "Should have found window.__FROM_STARTUP_CACHE__"
+    );
+    Assert.ok(
+      content.document.body.classList.contains("activity-stream"),
+      "Should have found activity-stream class on <body> element"
+    );
+    Assert.ok(
+      content.document.querySelector("[data-section-id='topsites']"),
+      "Should have found the Discovery Stream top sites."
+    );
+  });
+}
+
+/**
+ * Tests that the about:home document loaded in a passed <xul:browser> was
+ * dynamically generated, and _not_ from the cache.
+ *
+ * We test for this by looking for some tell-tale signs of the dynamically
+ * generated document:
+ *
+ *   1. No <script> elements (the scripts are loaded from the ScriptPreloader
+ *      via AboutNewTabChild when the "privileged about content process" is
+ *      enabled)
+ *   2. No __FROM_STARTUP_CACHE__ expando on the window
+ *   3. The "activity-stream" class on the document body
+ *   4. The top sites section
+ *
+ * @returns Promise
+ * @resolves undefined
+ *   Resolves once the cache entry has been destroyed.
+ */
+// eslint-disable-next-line no-unused-vars
+async function ensureDynamicAboutHome(browser) {
+  await SpecialPowers.spawn(browser, [], async () => {
+    let scripts = Array.from(content.document.querySelectorAll("script"));
+    Assert.equal(scripts.length, 0, "There should be no page scripts.");
+
+    Assert.equal(
+      Cu.waiveXrays(content).__FROM_STARTUP_CACHE__,
+      undefined,
+      "Should not have found window.__FROM_STARTUP_CACHE__"
+    );
+
+    Assert.ok(
+      content.document.body.classList.contains("activity-stream"),
+      "Should have found activity-stream class on <body> element"
+    );
+    Assert.ok(
+      content.document.querySelector("[data-section-id='topsites']"),
+      "Should have found the Discovery Stream top sites."
+    );
+  });
+}