Bug 1693301 - Nimbus test utility for automatic enrollment and cleanup r=k88hudson

Differential Revision: https://phabricator.services.mozilla.com/D106162

browser/components/newtab/test/browser/browser_aboutwelcome_multistage.js

@@ -232,23 +232,27 @@ add_task(async function setup() {
  */
 add_task(async function test_multistage_zeroOnboarding_experimentAPI() {
   await setAboutWelcomePref(true);
-  let updatePromise = ExperimentFakes.waitForExperimentUpdate(ExperimentAPI, {
-    slug: "mochitest-1-aboutwelcome",
-  });
-  ExperimentAPI._store.addExperiment({
-    slug: "mochitest-1-aboutwelcome",
-    branch: {
-      slug: "mochitest-1-aboutwelcome",
-      feature: {
-        enabled: false,
-        featureId: "aboutwelcome",
-        value: null,
-      },
-    },
-    active: true,
-  });
 
-  await updatePromise;
+  let {
+    enrollmentPromise,
+    doExperimentCleanup,
+  } = ExperimentFakes.enrollmentHelper(
+    ExperimentFakes.recipe("mochitest-1-aboutwelcome", {
+      branches: [
+        {
+          slug: "mochitest-1-aboutwelcome",
+          feature: {
+            enabled: false,
+            featureId: "aboutwelcome",
+            value: null,
+          },
+        },
+      ],
+      active: true,
+    })
+  );
+
+  await enrollmentPromise;
   ExperimentAPI._store._syncToChildren({ flush: true });
 
   let tab = await BrowserTestUtils.openNewForegroundTab(
@@ -256,6 +260,7 @@ add_task(async function test_multistage_zeroOnboarding_experimentAPI() {
     "about:welcome",
     true
   );
+
   registerCleanupFunction(() => {
     BrowserTestUtils.removeTab(tab);
   });
@@ -271,7 +276,7 @@ add_task(async function test_multistage_zeroOnboarding_experimentAPI() {
     ["div.onboardingContainer", "main.AW_STEP1"]
   );
 
-  ExperimentAPI._store._deleteForTests("mochitest-1-aboutwelcome");
+  await doExperimentCleanup();
   Assert.equal(ExperimentAPI._store.getAll().length, 0, "Cleanup done");
 });
 
@@ -280,24 +285,27 @@ add_task(async function test_multistage_zeroOnboarding_experimentAPI() {
  */
 add_task(async function test_multistage_aboutwelcome_experimentAPI() {
   await setAboutWelcomePref(true);
-  await setAboutWelcomeMultiStage({});
-  let updatePromise = ExperimentFakes.waitForExperimentUpdate(ExperimentAPI, {
-    slug: "mochitest-aboutwelcome",
-  });
-  ExperimentAPI._store.addExperiment({
-    slug: "mochitest-aboutwelcome",
-    branch: {
-      slug: "mochitest-aboutwelcome",
-      feature: {
-        enabled: true,
-        featureId: "aboutwelcome",
-        value: TEST_MULTISTAGE_CONTENT,
-      },
-    },
-    active: true,
-  });
 
-  await updatePromise;
+  let {
+    enrollmentPromise,
+    doExperimentCleanup,
+  } = ExperimentFakes.enrollmentHelper(
+    ExperimentFakes.recipe("mochitest-aboutwelcome", {
+      branches: [
+        {
+          slug: "mochitest-aboutwelcome-branch",
+          feature: {
+            enabled: true,
+            featureId: "aboutwelcome",
+            value: TEST_MULTISTAGE_CONTENT,
+          },
+        },
+      ],
+      active: true,
+    })
+  );
+
+  await enrollmentPromise;
   ExperimentAPI._store._syncToChildren({ flush: true });
 
   let tab = await BrowserTestUtils.openNewForegroundTab(
@@ -377,7 +385,7 @@ add_task(async function test_multistage_aboutwelcome_experimentAPI() {
     ["div.onboardingContainer"]
   );
 
-  ExperimentAPI._store._deleteForTests("mochitest-aboutwelcome");
+  await doExperimentCleanup();
   Assert.equal(ExperimentAPI._store.getAll().length, 0, "Cleanup done");
 });
 
@@ -819,23 +827,17 @@ add_task(async function test_AWMultistage_Import() {
   );
 });
 
-// Test events from AboutWelcomeUtils
-async function test_set_message() {
+add_task(async function test_onContentMessage() {
   Services.prefs.setBoolPref(DID_SEE_ABOUT_WELCOME_PREF, false);
+  await setAboutWelcomePref(true);
+
   await openAboutWelcome();
   Assert.equal(
     Services.prefs.getBoolPref(DID_SEE_ABOUT_WELCOME_PREF, false),
     true,
     "Pref was set"
   );
-}
-
-add_task(async function test_onContentMessage() {
-  await setAboutWelcomePref(true);
-  await openAboutWelcome();
-
-  //case "SET_WELCOME_MESSAGE_SEEN"
-  await test_set_message();
+  Services.prefs.clearUserPref(DID_SEE_ABOUT_WELCOME_PREF);
 });
 
 // Test Fxaccounts MetricsFlowURI

toolkit/components/nimbus/docs/index.rst

@@ -13,3 +13,4 @@ Learn more
   :maxdepth: 2
 
   integration.md
+  testing.md

toolkit/components/nimbus/docs/testing.md

@@ -0,0 +1,61 @@
+# Nimbus Testing Helpers
+
+In order to make testing easier we created some helpers that can be accessed by including
+
+```js
+const { ExperimentFakes } = ChromeUtils.import(
+  "resource://testing-common/NimbusTestUtils.jsm"
+);
+```
+
+## Testing your feature integrating with Nimbus
+
+1. You need to create a recipe
+
+```js
+let recipe = ExperimentFakes.recipe("my-cool-experiment", {
+  branches: [
+    {
+      slug: "treatment-branch",
+      ratio: 1,
+      feature: {
+        featureId: "<YOUR FEATURE>",
+        // The feature is on
+        enabled: true,
+        // If you defined `variables` in the MANIFEST
+        // the `value` should match that schema
+        value: null,
+      },
+    },
+  ],
+  bucketConfig: {
+    start: 0,
+    // Ensure 100% enrollment
+    count: 10000,
+    total: 10000,
+    namespace: "my-mochitest",
+    randomizationUnit: "normandy_id",
+  },
+});
+```
+
+2. Now with the newly created recipe you want the test to enroll in the experiment
+
+```js
+let {
+  enrollmentPromise,
+  doExperimentCleanup,
+} = ExperimentFakes.enrollmentHelper(recipe);
+
+// Await for enrollment to complete
+
+await enrollmentPromise;
+
+// Now you can assume the feature is enabled so you can
+// test and that it's doing the right thing
+
+// Assert.ok(It works!)
+
+// Finishing up
+await doExperimentCleanup();
+```

toolkit/components/nimbus/lib/ExperimentStore.jsm

@@ -179,8 +179,8 @@ class ExperimentStore extends SharedDataMap {
       );
     }
     this.set(experiment.slug, experiment);
-    this._emitExperimentUpdates(experiment);
     this._updateSyncStore(experiment);
+    this._emitExperimentUpdates(experiment);
   }
 
   /**

toolkit/components/nimbus/test/NimbusTestUtils.jsm

@@ -11,6 +11,7 @@ const { XPCOMUtils } = ChromeUtils.import(
 
 XPCOMUtils.defineLazyModuleGetters(this, {
   _ExperimentManager: "resource://nimbus/lib/ExperimentManager.jsm",
+  ExperimentManager: "resource://nimbus/lib/ExperimentManager.jsm",
   ExperimentStore: "resource://nimbus/lib/ExperimentStore.jsm",
   NormandyUtils: "resource://normandy/lib/NormandyUtils.jsm",
   FileTestUtils: "resource://testing-common/FileTestUtils.jsm",
@@ -67,6 +68,45 @@ const ExperimentFakes = {
 
     return new Promise(resolve => ExperimentAPI.on("update", options, resolve));
   },
+  enrollmentHelper(recipe = {}, { manager = ExperimentManager } = {}) {
+    let enrollmentPromise = new Promise(resolve =>
+      manager.store.on(`update:${recipe.slug}`, (event, experiment) => {
+        if (experiment.active) {
+          resolve(experiment);
+        }
+      })
+    );
+    let unenrollCompleted = slug =>
+      new Promise(resolve =>
+        manager.store.on(`update:${slug}`, (event, experiment) => {
+          if (!experiment.active) {
+            // Removes recipe from file storage which
+            // (normally the users archive of past experiments)
+            manager.store._deleteForTests(recipe.slug);
+            resolve();
+          }
+        })
+      );
+    let doExperimentCleanup = async () => {
+      for (let experiment of manager.store.getAllActive()) {
+        let promise = unenrollCompleted(experiment.slug);
+        manager.unenroll(experiment.slug, "cleanup");
+        await promise;
+      }
+      if (manager.store.getAllActive().length) {
+        throw new Error("Cleanup failed");
+      }
+    };
+
+    if (recipe.slug) {
+      if (!manager.store._isReady) {
+        throw new Error("Manager store not ready, call `manager.onStartup`");
+      }
+      manager.enroll(recipe);
+    }
+
+    return { enrollmentPromise, doExperimentCleanup };
+  },
   childStore() {
     return new ExperimentStore("FakeStore", { isParent: false });
   },

toolkit/components/nimbus/test/browser/browser.ini

@@ -4,6 +4,7 @@
 [browser_remotesettings_experiment_enroll.js]
 [browser_experiment_evaluate_jexl.js]
 [browser_remotesettingsexperimentloader_init.js]
+skip-if = true # Bug 1693437
 [browser_nimbus_telemetry.js]
 tags = remote-settings
 

toolkit/components/nimbus/test/browser/browser_remotesettingsexperimentloader_init.js

@@ -18,76 +18,71 @@ const { ExperimentFakes } = ChromeUtils.import(
 const { ExperimentManager } = ChromeUtils.import(
   "resource://nimbus/lib/ExperimentManager.jsm"
 );
+const { ExperimentAPI } = ChromeUtils.import(
+  "resource://nimbus/ExperimentAPI.jsm"
+);
 const { BrowserTestUtils } = ChromeUtils.import(
   "resource://testing-common/BrowserTestUtils.jsm"
 );
 const { sinon } = ChromeUtils.import("resource://testing-common/Sinon.jsm");
 
-let rsClient;
+function getRecipe(slug) {
+  return ExperimentFakes.recipe(slug, {
+    bucketConfig: {
+      start: 0,
+      // Make sure the experiment enrolls
+      count: 10000,
+      total: 10000,
+      namespace: "mochitest",
+      randomizationUnit: "normandy_id",
+    },
+    targeting: "!(experiment.slug in activeExperiments)",
+  });
+}
+
+let rsClient = RemoteSettings("nimbus-desktop-experiments");
 
 add_task(async function setup() {
+  RemoteSettingsExperimentLoader._updating = true;
   await SpecialPowers.pushPrefEnv({
     set: [
       ["messaging-system.log", "all"],
       ["app.shield.optoutstudies.enabled", true],
     ],
   });
-  rsClient = RemoteSettings("nimbus-desktop-experiments");
 
   registerCleanupFunction(async () => {
     await SpecialPowers.popPrefEnv();
-    await rsClient.db.clear();
   });
 });
 
 add_task(async function test_double_feature_enrollment() {
+  await rsClient.db.importChanges(
+    {},
+    42,
+    [getRecipe("foo" + Math.random()), getRecipe("foo" + Math.random())],
+    {
+      clear: true,
+    }
+  );
+  let { doExperimentCleanup } = ExperimentFakes.enrollmentHelper();
+  RemoteSettingsExperimentLoader.uninit();
+  await doExperimentCleanup();
   let sandbox = sinon.createSandbox();
-  // We want to prevent this because it would start a recipe
-  // update outside of our asserts
   sandbox.stub(RemoteSettingsExperimentLoader, "setTimer");
+  sandbox.stub(RemoteSettingsExperimentLoader, "onEnabledPrefChange");
   let sendFailureTelemetryStub = sandbox.stub(
     ExperimentManager,
     "sendFailureTelemetry"
   );
-
-  for (let experiment of ExperimentManager.store.getAllActive()) {
-    ExperimentManager.unenroll(experiment.slug, "cleanup");
-    ExperimentManager.store._deleteForTests(experiment.slug);
-  }
-  await BrowserTestUtils.waitForCondition(
-    () => ExperimentManager.store.getAllActive().length === 0
-  );
-
-  const recipe1 = ExperimentFakes.recipe("foo" + Date.now(), {
-    bucketConfig: {
-      start: 0,
-      // Make sure the experiment enrolls
-      count: 10000,
-      total: 10000,
-      namespace: "mochitest",
-      randomizationUnit: "normandy_id",
-    },
-  });
-  const recipe2 = ExperimentFakes.recipe("foo" + Date.now(), {
-    bucketConfig: {
-      start: 0,
-      // Make sure the experiment enrolls
-      count: 10000,
-      total: 10000,
-      namespace: "mochitest",
-      randomizationUnit: "normandy_id",
-    },
+  let enrolledPromise = ExperimentFakes.waitForExperimentUpdate(ExperimentAPI, {
+    featureId: "test-feature",
   });
 
-  await rsClient.db.importChanges({}, 42, [recipe1, recipe2], {
-    clear: true,
-  });
+  Assert.ok(ExperimentManager.store.getAllActive().length === 0, "Clean state");
 
-  RemoteSettingsExperimentLoader.uninit();
-  let enrolledPromise = new Promise(resolve =>
-    ExperimentManager.store.on("update:test-feature", resolve)
-  );
   await RemoteSettingsExperimentLoader.init();
+
   await enrolledPromise;
 
   Assert.ok(
@@ -95,18 +90,18 @@ add_task(async function test_double_feature_enrollment() {
     "It should initialize and process the recipes"
   );
 
-  BrowserTestUtils.waitForCondition(
-    () => sendFailureTelemetryStub.calledOnce,
+  Assert.equal(
+    ExperimentManager.store.getAllActive().length,
+    1,
+    "1 active experiment"
+  );
+
+  await BrowserTestUtils.waitForCondition(
+    () => sendFailureTelemetryStub.callCount,
     "Expected to fail one of the recipes"
   );
 
-  for (let experiment of ExperimentManager.store.getAllActive()) {
-    ExperimentManager.unenroll(experiment.slug, "cleanup");
-    ExperimentManager.store._deleteForTests(experiment.slug);
-  }
-  await BrowserTestUtils.waitForCondition(
-    () => ExperimentManager.store.getAllActive().length === 0
-  );
+  await doExperimentCleanup();
   await SpecialPowers.popPrefEnv();
   await rsClient.db.clear();
   sandbox.restore();

toolkit/components/nimbus/test/unit/test_NimbusTestUtils.js

@@ -11,3 +11,40 @@ add_task(async function test_recipe_fake_validates() {
     "should produce a valid experiment recipe"
   );
 });
+
+add_task(async function test_enrollmentHelper() {
+  let recipe = ExperimentFakes.recipe("bar");
+  recipe.branches.forEach(branch => {
+    // Use a feature that will set the sync pref cache
+    branch.feature.featureId = "aboutwelcome";
+  });
+  let manager = ExperimentFakes.manager();
+
+  await manager.onStartup();
+
+  let {
+    enrollmentPromise,
+    doExperimentCleanup,
+  } = ExperimentFakes.enrollmentHelper(recipe, { manager });
+
+  await enrollmentPromise;
+
+  Assert.ok(manager.store.getAllActive().length === 1, "Enrolled");
+  Assert.equal(
+    manager.store.getAllActive()[0].slug,
+    recipe.slug,
+    "Has expected slug"
+  );
+  Assert.ok(
+    Services.prefs.prefHasUserValue("nimbus.syncdatastore.aboutwelcome"),
+    "Sync pref cache set"
+  );
+
+  await doExperimentCleanup();
+
+  Assert.ok(manager.store.getAll().length === 0, "Cleanup done");
+  Assert.ok(
+    !Services.prefs.prefHasUserValue("nimbus.syncdatastore.aboutwelcome"),
+    "Sync pref cache is cleared"
+  );
+});