Bug 1602912 - Add Normandy graduation set r=Gijs

Add a mechanism to indicate that specific Normandy rollouts no longer apply to this version of Firefox. Normally this is handled by the preference specified by the rollout changing their built-in values to match the rollout values. However, this isn't always possible, such as if the preference is removed instead of being switch to on by default, or if the preference cannot be enabled by default for all users, but is conditionally enabled by another feature. Differential Revision: https://phabricator.services.mozilla.com/D102981
2021-01-29 00:28:05 +00:00 · 2021-01-29 00:28:05 +00:00 · e41395d88f
--- a/toolkit/components/normandy/actions/PreferenceRollbackAction.jsm
+++ b/toolkit/components/normandy/actions/PreferenceRollbackAction.jsm
@ -44,6 +44,23 @@ class PreferenceRollbackAction extends BaseAction {
    const { rolloutSlug } = recipe.arguments;
    const rollout = await PreferenceRollouts.get(rolloutSlug);

+    if (PreferenceRollouts.GRADUATION_SET.has(rolloutSlug)) {
+      // graduated rollouts can't be rolled back
+      TelemetryEvents.sendEvent(
+        "unenrollFailed",
+        "preference_rollback",
+        rolloutSlug,
+        {
+          reason: "in-graduation-set",
+          enrollmentId:
+            rollout?.enrollmentId ?? TelemetryEvents.NO_ENROLLMENT_ID_MARKER,
+        }
+      );
+      throw new Error(
+        `Cannot rollback rollout in graduation set "${rolloutSlug}".`
+      );
+    }
+
    if (!rollout) {
      this.log.debug(`Rollback ${rolloutSlug} not applicable, skipping`);
      return;
--- a/toolkit/components/normandy/actions/PreferenceRolloutAction.jsm
+++ b/toolkit/components/normandy/actions/PreferenceRolloutAction.jsm
@ -55,7 +55,15 @@ class PreferenceRolloutAction extends BaseAction {
  async _run(recipe) {
    const args = recipe.arguments;

-    // First determine which preferences are already being managed, to avoid
+    // Check if the rollout is on the list of rollouts to stop applying.
+    if (PreferenceRollouts.GRADUATION_SET.has(args.slug)) {
+      this.log.debug(
+        `Skipping rollout "${args.slug}" because it is in the graduation set.`
+      );
+      return;
+    }
+
+    // Determine which preferences are already being managed, to avoid
    // conflicts between recipes. This will throw if there is a problem.
    await this._verifyRolloutPrefs(args);

--- a/toolkit/components/normandy/docs/data-collection.rst
+++ b/toolkit/components/normandy/docs/data-collection.rst
@ -234,6 +234,90 @@ Unenroll Failed
         caused the attempted unenrollment.


+Preference Rollouts
+^^^^^^^^^^^^^^^^^^^
+Enrollment
+   Sent when a user first enrolls in a rollout.
+
+   method
+      The string ``"enroll"``
+   object
+      The string ``"preference_rollout"``
+   value
+      The slug of the rollout (``recipe.arguments.slug``)
+   extra
+      enrollmentId
+         A UUID that is unique to this user's enrollment in this rollout. It
+         will be included in all future telemetry for this user in this
+         rollout.
+
+Enroll Failed
+   Sent when a user attempts to enroll in a rollout, but the enrollment process fails.
+
+   method
+      The string ``"enrollFailed"``
+   object
+      The string ``"preference_rollout"``
+   value
+      The slug of the rollout (``recipe.arguments.slug``)
+   extra
+      reason
+         A code describing the reason the unenroll failed. Possible values are:
+
+         * ``"invalid type"``: The preferences specified in the rollout do not
+           match the preferences built in to the browser. The represents a
+           misconfiguration of the preferences in the recipe on the server.
+         * ``"would-be-no-op"``: All of the preference specified in the rollout
+           already have the given values. This represents an error in targeting
+           on the server.
+         * ``"conflict"``: At least one of the preferences specified in the
+           rollout is already managed by another active rollout.
+      preference
+         For ``reason="invalid type"``, the first preference that was invalid.
+         For ``reason="conflict"``, the first preference that is conflicting.
+
+Update
+   Sent when the preferences specified in the recipe have changed, and the
+   client updates the preferences of the browser to match.
+
+   method
+      The string ``"update"``
+   object
+      The string ``"preference_rollout"``
+   value
+      The slug of the rollout (``recipe.arguments.slug``)
+   extra
+      previousState
+         The state the rollout was in before this update (such as ``"active"`` or ``"graduated"``).
+      enrollmentId
+         The ID that was generated at enrollment.
+
+Graduation
+   Sent when Normandy determines that further intervention is no longer
+   needed for this rollout. After this point, Normandy will stop making
+   changes to the browser for this rollout, unless the rollout recipe changes
+   to specify preferences different than the built-in.
+
+   method
+      The string ``"graduate"``
+   object
+      The string ``"preference_rollout"``
+   value
+      The slug of the rollout (``recipe.arguments.slug``)
+   extra
+      reason
+         A code describing the reason for the graduation. Possible values are:
+
+         * ``"all-prefs-match"``: All preferences specified in the rollout now
+           have built-in values that match the rollouts values.
+           ``"in-graduation-set"``: The browser has changed versions (usually
+           updated) to one that specifies this rollout no longer applies and
+           should be graduated regardless of the built-in preference values.
+           This behavior is controlled by the constant
+           ``PreferenceRollouts.GRADUATION_SET``.
+      enrollmentId
+         The ID that was generated at enrollment.
+
 Add-on Studies
 ^^^^^^^^^^^^^^
 Enrollment
--- a/toolkit/components/normandy/lib/PreferenceExperiments.jsm
+++ b/toolkit/components/normandy/lib/PreferenceExperiments.jsm
@ -34,7 +34,6 @@
 *   Experiment branch that the user was matched to
 * @property {boolean} expired
 *   If false, the experiment is active.
- * @property {string} lastSeen
 *   ISO-formatted date string of when the experiment was last seen from the
 *   recipe server.
 * @property {string|null} temporaryErrorDeadline
--- a/toolkit/components/normandy/lib/PreferenceRollouts.jsm
+++ b/toolkit/components/normandy/lib/PreferenceRollouts.jsm
@ -125,13 +125,20 @@ var PreferenceRollouts = {
  STATE_ROLLED_BACK: "rolled-back",
  STATE_GRADUATED: "graduated",

+  // A set of rollout slugs that are obsolete based the code in this build of
+  // Firefox. This may include things like the preference no longer being
+  // applicable, or the feature changing in such a way that Normandy's automatic
+  // graduation system cannot detect that the rollout should hand off to the
+  // built-in code.
+  GRADUATION_SET: new Set(),
+
  /**
   * Update the rollout database with changes that happened during early startup.
   * @param {object} rolloutPrefsChanged Map from pref name to previous pref value
   */
  async recordOriginalValues(originalPreferences) {
    for (const rollout of await this.getAllActive()) {
-      let changed = false;
+      let shouldSaveRollout = false;

      // Count the number of preferences in this rollout that are now redundant.
      let prefMatchingDefaultCount = 0;
@ -146,28 +153,19 @@ var PreferenceRollouts = {
        // shut down), the correct value will be used.
        if (prefSpec.previousValue !== builtInDefault) {
          prefSpec.previousValue = builtInDefault;
-          changed = true;
+          shouldSaveRollout = true;
        }
      }

      if (prefMatchingDefaultCount === rollout.preferences.length) {
        // Firefox's builtin defaults have caught up to the rollout, making all
        // of the rollout's changes redundant, so graduate the rollout.
-        rollout.state = this.STATE_GRADUATED;
-        changed = true;
-        log.debug(`Graduating rollout: ${rollout.slug}`);
-        TelemetryEvents.sendEvent(
-          "graduate",
-          "preference_rollout",
-          rollout.slug,
-          {
-            enrollmentId:
-              rollout.enrollmentId || TelemetryEvents.NO_ENROLLMENT_ID_MARKER,
-          }
-        );
+        await this.graduate(rollout, "all-prefs-match");
+        // `this.graduate` writes the rollout to the db, so we don't need to do it anymore.
+        shouldSaveRollout = false;
      }

-      if (changed) {
+      if (shouldSaveRollout) {
        const db = await getDatabase();
        await getStore(db, "readwrite").put(rollout);
      }
@ -178,6 +176,10 @@ var PreferenceRollouts = {
    CleanupManager.addCleanupHandler(() => this.saveStartupPrefs());

    for (const rollout of await this.getAllActive()) {
+      if (this.GRADUATION_SET.has(rollout.slug)) {
+        await this.graduate(rollout, "in-graduation-set");
+        continue;
+      }
      TelemetryEnvironment.setExperimentActive(rollout.slug, rollout.state, {
        type: "normandy-prefrollout",
        enrollmentId:
@ -199,19 +201,26 @@ var PreferenceRollouts = {
   * Test wrapper that temporarily replaces the stored rollout data with fake
   * data for testing.
   */
-  withTestMock(testFunction) {
-    return async function inner(...args) {
-      let db = await getDatabase();
-      const oldData = await getStore(db, "readonly").getAll();
-      await getStore(db, "readwrite").clear();
-      try {
-        await testFunction(...args);
-      } finally {
-        db = await getDatabase();
+  withTestMock({ graduationSet = new Set(), rollouts = [] } = {}) {
+    return testFunction => {
+      return async (...args) => {
+        let db = await getDatabase();
+        const oldData = await getStore(db, "readonly").getAll();
        await getStore(db, "readwrite").clear();
-        const store = getStore(db, "readwrite");
-        await Promise.all(oldData.map(d => store.add(d)));
-      }
+        await Promise.all(rollouts.map(r => this.add(r)));
+        const oldGraduationSet = this.GRADUATION_SET;
+        this.GRADUATION_SET = graduationSet;
+
+        try {
+          await testFunction(...args);
+        } finally {
+          this.GRADUATION_SET = oldGraduationSet;
+          db = await getDatabase();
+          await getStore(db, "readwrite").clear();
+          const store = getStore(db, "readwrite");
+          await Promise.all(oldData.map(d => store.add(d)));
+        }
+      };
    };
  },

@ -328,4 +337,16 @@ var PreferenceRollouts = {
      }
    }
  },
+
+  async graduate(rollout, reason) {
+    log.debug(`Graduating rollout: ${rollout.slug}`);
+    rollout.state = this.STATE_GRADUATED;
+    const db = await getDatabase();
+    await getStore(db, "readwrite").put(rollout);
+    TelemetryEvents.sendEvent("graduate", "preference_rollout", rollout.slug, {
+      reason,
+      enrollmentId:
+        rollout.enrollmentId || TelemetryEvents.NO_ENROLLMENT_ID_MARKER,
+    });
+  },
 };
--- a/toolkit/components/normandy/test/browser/browser_AddonStudies.js
+++ b/toolkit/components/normandy/test/browser/browser_AddonStudies.js
@ -108,7 +108,7 @@ decorate_task(
      studyEndDate: new Date(2012, 1),
    }),
  ]),
-  withSendEventStub,
+  withSendEventSpy,
  withInstalledWebExtension(
    { id: "installed@example.com" },
    /* expectUninstall: */ true
--- a/toolkit/components/normandy/test/browser/browser_ClientEnvironment.js
+++ b/toolkit/components/normandy/test/browser/browser_ClientEnvironment.js
@ -208,7 +208,7 @@ decorate_task(
  }
 );

-decorate_task(PreferenceRollouts.withTestMock, async function testRollouts() {
+decorate_task(PreferenceRollouts.withTestMock(), async function testRollouts() {
  const prefRollout = {
    slug: "test-rollout",
    preference: [],
--- a/toolkit/components/normandy/test/browser/browser_Normandy.js
+++ b/toolkit/components/normandy/test/browser/browser_Normandy.js
@ -299,7 +299,7 @@ decorate_task(
  AddonStudies.withStudies([
    factories.addonStudyFactory({ slug: "test-study" }),
  ]),
-  PreferenceRollouts.withTestMock,
+  PreferenceRollouts.withTestMock(),
  AddonRollouts.withTestMock,
  async function disablingTelemetryClearsEnrollmentIds(
    [prefExperiment],
--- a/toolkit/components/normandy/test/browser/browser_PreferenceExperiments.js
+++ b/toolkit/components/normandy/test/browser/browser_PreferenceExperiments.js
@ -320,7 +320,7 @@ decorate_task(
 // start should throw if an experiment with the given name already exists
 decorate_task(
  withMockExperiments([preferenceStudyFactory({ slug: "test" })]),
-  withSendEventStub,
+  withSendEventSpy,
  async function(experiments, sendEventStub) {
    await Assert.rejects(
      PreferenceExperiments.start({
@ -354,7 +354,7 @@ decorate_task(
      preferences: { "fake.preferenceinteger": {} },
    }),
  ]),
-  withSendEventStub,
+  withSendEventSpy,
  async function(experiments, sendEventStub) {
    await Assert.rejects(
      PreferenceExperiments.start({
@ -390,7 +390,7 @@ decorate_task(
 );

 // start should throw if an invalid preferenceBranchType is given
-decorate_task(withMockExperiments(), withSendEventStub, async function(
+decorate_task(withMockExperiments(), withSendEventSpy, async function(
  experiments,
  sendEventStub
 ) {
@ -422,7 +422,7 @@ decorate_task(
  withMockExperiments(),
  withMockPreferences,
  withStub(PreferenceExperiments, "startObserver"),
-  withSendEventStub,
+  withSendEventSpy,
  async function testStart(
    experiments,
    mockPreferences,
@ -587,7 +587,7 @@ decorate_task(
 );

 // start should detect if a new preference value type matches the previous value type
-decorate_task(withMockPreferences, withSendEventStub, async function(
+decorate_task(withMockPreferences, withSendEventSpy, async function(
  mockPreferences,
  sendEventStub
 ) {
@ -856,7 +856,7 @@ decorate_task(
 );

 // stop should throw if an experiment with the given name doesn't exist
-decorate_task(withMockExperiments(), withSendEventStub, async function(
+decorate_task(withMockExperiments(), withSendEventSpy, async function(
  experiments,
  sendEventStub
 ) {
@ -881,7 +881,7 @@ decorate_task(
  withMockExperiments([
    preferenceStudyFactory({ slug: "test", expired: true }),
  ]),
-  withSendEventStub,
+  withSendEventSpy,
  async function(experiments, sendEventStub) {
    await Assert.rejects(
      PreferenceExperiments.stop("test"),
@ -920,7 +920,7 @@ decorate_task(
  ]),
  withMockPreferences,
  withSpy(PreferenceExperiments, "stopObserver"),
-  withSendEventStub,
+  withSendEventSpy,
  async function testStop(
    experiments,
    mockPreferences,
@ -1074,7 +1074,7 @@ decorate_task(
  ]),
  withMockPreferences,
  withStub(PreferenceExperiments, "stopObserver"),
-  withSendEventStub,
+  withSendEventSpy,
  async function testStopReset(
    experiments,
    mockPreferences,
@ -1289,7 +1289,7 @@ decorate_task(
  withMockExperiments(),
  withStub(TelemetryEnvironment, "setExperimentActive"),
  withStub(TelemetryEnvironment, "setExperimentInactive"),
-  withSendEventStub,
+  withSendEventSpy,
  async function testStartAndStopTelemetry(
    experiments,
    setActiveStub,
@ -1357,7 +1357,7 @@ decorate_task(
  withMockExperiments(),
  withStub(TelemetryEnvironment, "setExperimentActive"),
  withStub(TelemetryEnvironment, "setExperimentInactive"),
-  withSendEventStub,
+  withSendEventSpy,
  async function testInitTelemetryExperimentType(
    experiments,
    setActiveStub,
@ -1766,7 +1766,7 @@ decorate_task(
  ]),
  withMockPreferences,
  withStub(PreferenceExperiments, "stopObserver"),
-  withSendEventStub,
+  withSendEventSpy,
  async function testStopUnknownReason(
    experiments,
    mockPreferences,
@ -1807,7 +1807,7 @@ decorate_task(
  ]),
  withMockPreferences,
  withStub(PreferenceExperiments, "stopObserver"),
-  withSendEventStub,
+  withSendEventSpy,
  async function testStopResetValue(
    experiments,
    mockPreferences,
@ -1838,7 +1838,7 @@ decorate_task(
 // the user changed preferences during a browser run.
 decorate_task(
  withMockPreferences,
-  withSendEventStub,
+  withSendEventSpy,
  withMockExperiments([
    preferenceStudyFactory({
      slug: "test",
--- a/toolkit/components/normandy/test/browser/browser_PreferenceRollouts.js
+++ b/toolkit/components/normandy/test/browser/browser_PreferenceRollouts.js
@ -5,15 +5,18 @@ ChromeUtils.import("resource://gre/modules/TelemetryEnvironment.jsm", this);
 ChromeUtils.import("resource://normandy/lib/PreferenceRollouts.jsm", this);
 ChromeUtils.import("resource://normandy/lib/TelemetryEvents.jsm", this);

-decorate_task(PreferenceRollouts.withTestMock, async function testGetMissing() {
-  ok(
-    !(await PreferenceRollouts.get("does-not-exist")),
-    "get should return null when the requested rollout does not exist"
-  );
-});
+decorate_task(
+  PreferenceRollouts.withTestMock(),
+  async function testGetMissing() {
+    ok(
+      !(await PreferenceRollouts.get("does-not-exist")),
+      "get should return null when the requested rollout does not exist"
+    );
+  }
+);

 decorate_task(
-  PreferenceRollouts.withTestMock,
+  PreferenceRollouts.withTestMock(),
  async function testAddUpdateAndGet() {
    const rollout = {
      slug: "test-rollout",
@ -41,7 +44,7 @@ decorate_task(
 );

 decorate_task(
-  PreferenceRollouts.withTestMock,
+  PreferenceRollouts.withTestMock(),
  async function testCantUpdateNonexistent() {
    const rollout = {
      slug: "test-rollout",
@ -60,7 +63,7 @@ decorate_task(
  }
 );

-decorate_task(PreferenceRollouts.withTestMock, async function testGetAll() {
+decorate_task(PreferenceRollouts.withTestMock(), async function testGetAll() {
  const rollout1 = {
    slug: "test-rollout-1",
    preference: [],
@ -83,7 +86,7 @@ decorate_task(PreferenceRollouts.withTestMock, async function testGetAll() {
 });

 decorate_task(
-  PreferenceRollouts.withTestMock,
+  PreferenceRollouts.withTestMock(),
  async function testGetAllActive() {
    const rollout1 = {
      slug: "test-rollout-1",
@ -113,7 +116,7 @@ decorate_task(
  }
 );

-decorate_task(PreferenceRollouts.withTestMock, async function testHas() {
+decorate_task(PreferenceRollouts.withTestMock(), async function testHas() {
  const rollout = {
    slug: "test-rollout",
    preferences: [],
@ -132,7 +135,7 @@ decorate_task(PreferenceRollouts.withTestMock, async function testHas() {

 // recordOriginalValue should update storage to note the original values
 decorate_task(
-  PreferenceRollouts.withTestMock,
+  PreferenceRollouts.withTestMock(),
  async function testRecordOriginalValuesUpdatesPreviousValues() {
    await PreferenceRollouts.add({
      slug: "test-rollout",
@ -162,11 +165,11 @@ decorate_task(
  }
 );

-// recordOriginalValue should graduate a study when it is no longer relevant.
+// recordOriginalValue should graduate a study when all of its preferences are built-in
 decorate_task(
-  PreferenceRollouts.withTestMock,
-  withSendEventStub,
-  async function testRecordOriginalValuesUpdatesPreviousValues(sendEventStub) {
+  PreferenceRollouts.withTestMock(),
+  withSendEventSpy,
+  async function testRecordOriginalValuesGraduates(sendEventStub) {
    await PreferenceRollouts.add({
      slug: "test-rollout",
      state: PreferenceRollouts.STATE_ACTIVE,
@ -216,7 +219,7 @@ decorate_task(

 // init should mark active rollouts in telemetry
 decorate_task(
-  PreferenceRollouts.withTestMock,
+  PreferenceRollouts.withTestMock(),
  withStub(TelemetryEnvironment, "setExperimentActive"),
  async function testInitTelemetry(setExperimentActiveStub) {
    await PreferenceRollouts.add({
@ -260,3 +263,41 @@ decorate_task(
    );
  }
 );
+
+// init should graduate rollouts in the graduation set
+decorate_task(
+  withStub(TelemetryEnvironment, "setExperimentActive"),
+  withSendEventSpy,
+  PreferenceRollouts.withTestMock({
+    graduationSet: new Set(["test-rollout"]),
+    rollouts: [
+      {
+        slug: "test-rollout",
+        state: PreferenceRollouts.STATE_ACTIVE,
+        enrollmentId: "test-enrollment-id",
+      },
+    ],
+  }),
+  async function testInitGraduationSet(setExperimentActiveStub, sendEventStub) {
+    await PreferenceRollouts.init();
+    const newRollout = await PreferenceRollouts.get("test-rollout");
+    Assert.equal(
+      newRollout.state,
+      PreferenceRollouts.STATE_GRADUATED,
+      "the rollout should be graduated"
+    );
+    Assert.deepEqual(
+      setExperimentActiveStub.args,
+      [],
+      "setExperimentActive should not be called"
+    );
+    sendEventStub.assertEvents([
+      [
+        "graduate",
+        "preference_rollout",
+        "test-rollout",
+        { enrollmentId: "test-enrollment-id", reason: "in-graduation-set" },
+      ],
+    ]);
+  }
+).only();
--- a/toolkit/components/normandy/test/browser/browser_actions_AddonRollbackAction.js
+++ b/toolkit/components/normandy/test/browser/browser_actions_AddonRollbackAction.js
@ -15,7 +15,7 @@ decorate_task(
  ensureAddonCleanup,
  withMockNormandyApi,
  withStub(TelemetryEnvironment, "setExperimentInactive"),
-  withSendEventStub,
+  withSendEventSpy,
  async function simple_recipe_unenrollment(
    mockApi,
    setExperimentInactiveStub,
@ -109,7 +109,7 @@ decorate_task(
  AddonRollouts.withTestMock,
  ensureAddonCleanup,
  withMockNormandyApi,
-  withSendEventStub,
+  withSendEventSpy,
  async function addon_already_uninstalled(mockApi, sendEventStub) {
    const rolloutRecipe = {
      id: 1,
@ -192,7 +192,7 @@ decorate_task(
  AddonRollouts.withTestMock,
  ensureAddonCleanup,
  withMockNormandyApi,
-  withSendEventStub,
+  withSendEventSpy,
  async function already_rolled_back(mockApi, sendEventStub) {
    const rollout = {
      recipeId: 1,
--- a/toolkit/components/normandy/test/browser/browser_actions_AddonRolloutAction.js
+++ b/toolkit/components/normandy/test/browser/browser_actions_AddonRolloutAction.js
@ -14,7 +14,7 @@ decorate_task(
  ensureAddonCleanup,
  withMockNormandyApi,
  withStub(TelemetryEnvironment, "setExperimentActive"),
-  withSendEventStub,
+  withSendEventSpy,
  async function simple_recipe_enrollment(
    mockApi,
    setExperimentActiveStub,
@ -92,7 +92,7 @@ decorate_task(
  AddonRollouts.withTestMock,
  ensureAddonCleanup,
  withMockNormandyApi,
-  withSendEventStub,
+  withSendEventSpy,
  async function update_rollout(mockApi, sendEventStub) {
    // first enrollment
    const recipe = {
@ -184,7 +184,7 @@ decorate_task(
  AddonRollouts.withTestMock,
  ensureAddonCleanup,
  withMockNormandyApi,
-  withSendEventStub,
+  withSendEventSpy,
  async function rerun_recipe(mockApi, sendEventStub) {
    const recipe = {
      id: 1,
@ -260,7 +260,7 @@ decorate_task(
  AddonRollouts.withTestMock,
  ensureAddonCleanup,
  withMockNormandyApi,
-  withSendEventStub,
+  withSendEventSpy,
  async function conflicting_rollout(mockApi, sendEventStub) {
    const recipe = {
      id: 1,
@ -356,7 +356,7 @@ decorate_task(
  AddonRollouts.withTestMock,
  ensureAddonCleanup,
  withMockNormandyApi,
-  withSendEventStub,
+  withSendEventSpy,
  async function enroll_failed_addon_id_changed(mockApi, sendEventStub) {
    const recipe = {
      id: 1,
@ -448,7 +448,7 @@ decorate_task(
  AddonRollouts.withTestMock,
  ensureAddonCleanup,
  withMockNormandyApi,
-  withSendEventStub,
+  withSendEventSpy,
  async function enroll_failed_upgrade_required(mockApi, sendEventStub) {
    const recipe = {
      id: 1,
--- a/toolkit/components/normandy/test/browser/browser_actions_BranchedAddonStudyAction.js
+++ b/toolkit/components/normandy/test/browser/browser_actions_BranchedAddonStudyAction.js
@ -81,7 +81,7 @@ decorate_task(
  ensureAddonCleanup,
  withMockNormandyApi,
  AddonStudies.withStudies([branchedAddonStudyFactory()]),
-  withSendEventStub,
+  withSendEventSpy,
  withInstalledWebExtensionSafe({ id: FIXTURE_ADDON_ID, version: "1.0" }),
  async function enrollTwiceFail(
    mockApi,
@ -114,7 +114,7 @@ decorate_task(
  withStudiesEnabled,
  ensureAddonCleanup,
  withMockNormandyApi,
-  withSendEventStub,
+  withSendEventSpy,
  AddonStudies.withStudies(),
  async function enrollDownloadFail(mockApi, sendEventStub) {
    const recipe = branchedAddonStudyRecipeFactory({
@ -155,7 +155,7 @@ decorate_task(
  withStudiesEnabled,
  ensureAddonCleanup,
  withMockNormandyApi,
-  withSendEventStub,
+  withSendEventSpy,
  AddonStudies.withStudies(),
  async function enrollHashCheckFails(mockApi, sendEventStub) {
    const recipe = branchedAddonStudyRecipeFactory();
@ -193,7 +193,7 @@ decorate_task(
  withStudiesEnabled,
  ensureAddonCleanup,
  withMockNormandyApi,
-  withSendEventStub,
+  withSendEventSpy,
  AddonStudies.withStudies(),
  async function enrollFailsMetadataMismatch(mockApi, sendEventStub) {
    const recipe = branchedAddonStudyRecipeFactory();
@ -231,7 +231,7 @@ decorate_task(
  withStudiesEnabled,
  ensureAddonCleanup,
  withMockNormandyApi,
-  withSendEventStub,
+  withSendEventSpy,
  withInstalledWebExtensionSafe({ version: "0.1", id: FIXTURE_ADDON_ID }),
  AddonStudies.withStudies(),
  async function conflictingEnrollment(
@ -290,7 +290,7 @@ decorate_task(
      addonVersion: "1.0",
    }),
  ]),
-  withSendEventStub,
+  withSendEventSpy,
  withInstalledWebExtensionSafe({ id: FIXTURE_ADDON_ID, version: "1.0" }),
  async function successfulUpdate(
    mockApi,
@ -368,7 +368,7 @@ decorate_task(
      addonVersion: "0.1",
    }),
  ]),
-  withSendEventStub,
+  withSendEventSpy,
  withInstalledWebExtensionSafe({ id: FIXTURE_ADDON_ID, version: "0.1" }),
  async function updateFailsAddonIdMismatch(
    mockApi,
@ -422,7 +422,7 @@ decorate_task(
      addonVersion: "0.1",
    }),
  ]),
-  withSendEventStub,
+  withSendEventSpy,
  withInstalledWebExtensionSafe({ id: "test@example.com", version: "0.1" }),
  async function updateFailsAddonDoesNotExist(
    mockApi,
@ -480,7 +480,7 @@ decorate_task(
      addonVersion: "0.1",
    }),
  ]),
-  withSendEventStub,
+  withSendEventSpy,
  withInstalledWebExtensionSafe({ id: FIXTURE_ADDON_ID, version: "0.1" }),
  async function updateDownloadFailure(
    mockApi,
@ -536,7 +536,7 @@ decorate_task(
      addonVersion: "0.1",
    }),
  ]),
-  withSendEventStub,
+  withSendEventSpy,
  withInstalledWebExtensionSafe({ id: FIXTURE_ADDON_ID, version: "0.1" }),
  async function updateFailsHashCheckFail(
    mockApi,
@ -593,7 +593,7 @@ decorate_task(
      addonVersion: "2.0",
    }),
  ]),
-  withSendEventStub,
+  withSendEventSpy,
  withInstalledWebExtensionSafe({ id: FIXTURE_ADDON_ID, version: "2.0" }),
  async function upgradeFailsNoDowngrades(
    mockApi,
@ -649,7 +649,7 @@ decorate_task(
      addonVersion: "1.0",
    }),
  ]),
-  withSendEventStub,
+  withSendEventSpy,
  withInstalledWebExtensionFromURL(
    FIXTURE_ADDON_DETAILS["normandydriver-a-1.0"].url
  ),
@ -726,7 +726,7 @@ decorate_task(
  withStudiesEnabled,
  ensureAddonCleanup,
  AddonStudies.withStudies([branchedAddonStudyFactory({ active: false })]),
-  withSendEventStub,
+  withSendEventSpy,
  async ([study], sendEventStub) => {
    const action = new BranchedAddonStudyAction();
    await Assert.rejects(
@ -750,7 +750,7 @@ decorate_task(
    }),
  ]),
  withInstalledWebExtension({ id: testStopId }, /* expectUninstall: */ true),
-  withSendEventStub,
+  withSendEventSpy,
  withStub(TelemetryEnvironment, "setExperimentInactive"),
  async function unenrollTest(
    [study],
@ -803,7 +803,7 @@ decorate_task(
      addonId: "missingAddon@example.com",
    }),
  ]),
-  withSendEventStub,
+  withSendEventSpy,
  async function unenrollMissingAddonTest([study], sendEventStub) {
    const action = new BranchedAddonStudyAction();

@ -832,7 +832,7 @@ decorate_task(
  withStudiesEnabled,
  ensureAddonCleanup,
  withMockNormandyApi,
-  withSendEventStub,
+  withSendEventSpy,
  withMockPreferences,
  AddonStudies.withStudies(),
  async function testOptOut(mockApi, sendEventStub, mockPreferences) {
@ -868,7 +868,7 @@ decorate_task(
  withStudiesEnabled,
  ensureAddonCleanup,
  withMockNormandyApi,
-  withSendEventStub,
+  withSendEventSpy,
  AddonStudies.withStudies(),
  async function testEnrollmentPaused(mockApi, sendEventStub) {
    const action = new BranchedAddonStudyAction();
@ -908,7 +908,7 @@ decorate_task(
      addonVersion: "1.0",
    }),
  ]),
-  withSendEventStub,
+  withSendEventSpy,
  withInstalledWebExtensionSafe({ id: FIXTURE_ADDON_ID, version: "1.0" }),
  async function testUpdateEnrollmentPaused(
    mockApi,
@ -978,7 +978,7 @@ const successEnrollBranchedTest = decorate(
  withStudiesEnabled,
  ensureAddonCleanup,
  withMockNormandyApi,
-  withSendEventStub,
+  withSendEventSpy,
  withStub(TelemetryEnvironment, "setExperimentActive"),
  AddonStudies.withStudies(),
  async function(branch, mockApi, sendEventStub, setExperimentActiveStub) {
@ -1117,7 +1117,7 @@ decorate_task(
  ensureAddonCleanup,
  withMockNormandyApi,
  AddonStudies.withStudies([branchedAddonStudyFactory()]),
-  withSendEventStub,
+  withSendEventSpy,
  withInstalledWebExtensionSafe(
    { id: FIXTURE_ADDON_ID, version: "1.0" },
    /* expectUninstall: */ true
@ -1190,7 +1190,7 @@ decorate_task(
  withStudiesEnabled,
  ensureAddonCleanup,
  withMockNormandyApi,
-  withSendEventStub,
+  withSendEventSpy,
  AddonStudies.withStudies(),
  async function noAddonBranches(mockApi, sendEventStub) {
    const initialAddonIds = (await AddonManager.getAllAddons()).map(
--- a/toolkit/components/normandy/test/browser/browser_actions_PreferenceRollbackAction.js
+++ b/toolkit/components/normandy/test/browser/browser_actions_PreferenceRollbackAction.js
@ -14,9 +14,9 @@ ChromeUtils.import("resource://normandy/lib/TelemetryEvents.jsm", this);

 // Test that a simple recipe rollsback as expected
 decorate_task(
-  PreferenceRollouts.withTestMock,
+  PreferenceRollouts.withTestMock(),
  withStub(TelemetryEnvironment, "setExperimentInactive"),
-  withSendEventStub,
+  withSendEventSpy,
  async function simple_rollback(setExperimentInactiveStub, sendEventStub) {
    Services.prefs.getDefaultBranch("").setIntPref("test.pref1", 2);
    Services.prefs
@ -127,8 +127,8 @@ decorate_task(

 // Test that a graduated rollout can't be rolled back
 decorate_task(
-  PreferenceRollouts.withTestMock,
-  withSendEventStub,
+  PreferenceRollouts.withTestMock(),
+  withSendEventSpy,
  async function cant_rollback_graduated(sendEventStub) {
    Services.prefs.getDefaultBranch("").setIntPref("test.pref", 1);
    await PreferenceRollouts.add({
@ -186,8 +186,8 @@ decorate_task(

 // Test that a rollback without a matching rollout does not send telemetry
 decorate_task(
-  PreferenceRollouts.withTestMock,
-  withSendEventStub,
+  PreferenceRollouts.withTestMock(),
+  withSendEventSpy,
  withStub(Uptake, "reportRecipe"),
  async function rollback_without_rollout(sendEventStub, reportRecipeStub) {
    let recipe = { id: 1, arguments: { rolloutSlug: "missing-rollout" } };
@ -208,9 +208,9 @@ decorate_task(

 // Test that rolling back an already rolled back recipe doesn't do anything
 decorate_task(
-  PreferenceRollouts.withTestMock,
+  PreferenceRollouts.withTestMock(),
  withStub(TelemetryEnvironment, "setExperimentInactive"),
-  withSendEventStub,
+  withSendEventSpy,
  async function rollback_already_rolled_back(
    setExperimentInactiveStub,
    sendEventStub
@ -261,7 +261,7 @@ decorate_task(
 );

 // Test that a rollback doesn't affect user prefs
-decorate_task(PreferenceRollouts.withTestMock, async function simple_rollback(
+decorate_task(PreferenceRollouts.withTestMock(), async function simple_rollback(
  setExperimentInactiveStub,
  sendEventStub
 ) {
@ -303,3 +303,50 @@ decorate_task(PreferenceRollouts.withTestMock, async function simple_rollback(
  Services.prefs.deleteBranch("test.pref");
  Services.prefs.getDefaultBranch("").deleteBranch("test.pref");
 });
+
+// Test that a rollouts in the graduation set can't be rolled back
+decorate_task(
+  PreferenceRollouts.withTestMock({
+    graduationSet: new Set(["graduated-rollout"]),
+  }),
+  withSendEventSpy,
+  async function cant_rollback_graduation_set(sendEventStub) {
+    Services.prefs.getDefaultBranch("").setIntPref("test.pref", 1);
+
+    let recipe = { id: 1, arguments: { rolloutSlug: "graduated-rollout" } };
+
+    const action = new PreferenceRollbackAction();
+    await action.processRecipe(recipe, BaseAction.suitability.FILTER_MATCH);
+    await action.finalize();
+    is(action.lastError, null, "lastError should be null");
+
+    is(Services.prefs.getIntPref("test.pref"), 1, "pref should not change");
+    is(
+      Services.prefs.getPrefType("app.normandy.startupRolloutPrefs.test.pref"),
+      Services.prefs.PREF_INVALID,
+      "no startup pref should be added"
+    );
+
+    // No entry in the DB
+    Assert.deepEqual(
+      await PreferenceRollouts.getAll(),
+      [],
+      "Rollout should be in the db"
+    );
+
+    sendEventStub.assertEvents([
+      [
+        "unenrollFailed",
+        "preference_rollback",
+        "graduated-rollout",
+        {
+          reason: "in-graduation-set",
+          enrollmentId: TelemetryEvents.NO_ENROLLMENT_ID,
+        },
+      ],
+    ]);
+
+    // Cleanup
+    Services.prefs.getDefaultBranch("").deleteBranch("test.pref");
+  }
+);
--- a/toolkit/components/normandy/test/browser/browser_actions_PreferenceRolloutAction.js
+++ b/toolkit/components/normandy/test/browser/browser_actions_PreferenceRolloutAction.js
@ -14,9 +14,9 @@ ChromeUtils.import("resource://testing-common/NormandyTestUtils.jsm", this);

 // Test that a simple recipe enrolls as expected
 decorate_task(
-  PreferenceRollouts.withTestMock,
+  PreferenceRollouts.withTestMock(),
  withStub(TelemetryEnvironment, "setExperimentActive"),
-  withSendEventStub,
+  withSendEventSpy,
  async function simple_recipe_enrollment(
    setExperimentActiveStub,
    sendEventStub
@ -124,8 +124,8 @@ decorate_task(

 // Test that an enrollment's values can change, be removed, and be added
 decorate_task(
-  PreferenceRollouts.withTestMock,
-  withSendEventStub,
+  PreferenceRollouts.withTestMock(),
+  withSendEventSpy,
  async function update_enrollment(sendEventStub) {
    // first enrollment
    const recipe = {
@ -237,8 +237,8 @@ decorate_task(

 // Test that a graduated rollout can be ungraduated
 decorate_task(
-  PreferenceRollouts.withTestMock,
-  withSendEventStub,
+  PreferenceRollouts.withTestMock(),
+  withSendEventSpy,
  async function ungraduate_enrollment(sendEventStub) {
    Services.prefs.getDefaultBranch("").setIntPref("test.pref", 1);
    await PreferenceRollouts.add({
@ -302,8 +302,8 @@ decorate_task(

 // Test when recipes conflict, only one is applied
 decorate_task(
-  PreferenceRollouts.withTestMock,
-  withSendEventStub,
+  PreferenceRollouts.withTestMock(),
+  withSendEventSpy,
  async function conflicting_recipes(sendEventStub) {
    // create two recipes that each share a pref and have a unique pref.
    const recipe1 = {
@ -415,8 +415,8 @@ decorate_task(

 // Test when the wrong value type is given, the recipe is not applied
 decorate_task(
-  PreferenceRollouts.withTestMock,
-  withSendEventStub,
+  PreferenceRollouts.withTestMock(),
+  withSendEventSpy,
  async function wrong_preference_value(sendEventStub) {
    Services.prefs.getDefaultBranch("").setCharPref("test.pref", "not an int");
    const recipe = {
@ -464,7 +464,7 @@ decorate_task(

 // Test that even when applying a rollout, user prefs are preserved
 decorate_task(
-  PreferenceRollouts.withTestMock,
+  PreferenceRollouts.withTestMock(),
  async function preserves_user_prefs() {
    Services.prefs
      .getDefaultBranch("")
@ -522,7 +522,7 @@ decorate_task(

 // Enrollment works for prefs with only a user branch value, and no default value.
 decorate_task(
-  PreferenceRollouts.withTestMock,
+  PreferenceRollouts.withTestMock(),
  async function simple_recipe_enrollment() {
    const recipe = {
      id: 1,
@ -564,8 +564,8 @@ decorate_task(
 // When running a rollout a second time on a pref that doesn't have an existing
 // value, the previous value is handled correctly.
 decorate_task(
-  PreferenceRollouts.withTestMock,
-  withSendEventStub,
+  PreferenceRollouts.withTestMock(),
+  withSendEventSpy,
  async function(sendEventStub) {
    const recipe = {
      id: 1,
@ -617,9 +617,9 @@ decorate_task(

 // New rollouts that are no-ops should send errors
 decorate_task(
-  PreferenceRollouts.withTestMock,
+  PreferenceRollouts.withTestMock(),
  withStub(TelemetryEnvironment, "setExperimentActive"),
-  withSendEventStub,
+  withSendEventSpy,
  async function no_op_new_recipe(setExperimentActiveStub, sendEventStub) {
    Services.prefs.getDefaultBranch("").setIntPref("test.pref", 1);

@ -670,3 +670,55 @@ decorate_task(
    Services.prefs.getDefaultBranch("").deleteBranch("test.pref");
  }
 );
+
+// New rollouts in the graduation set should silently do nothing
+decorate_task(
+  withStub(TelemetryEnvironment, "setExperimentActive"),
+  withSendEventSpy,
+  PreferenceRollouts.withTestMock({ graduationSet: new Set(["test-rollout"]) }),
+  async function graduationSetNewRecipe(
+    setExperimentActiveStub,
+    sendEventStub
+  ) {
+    Services.prefs.getDefaultBranch("").setIntPref("test.pref", 1);
+
+    const recipe = {
+      id: 1,
+      arguments: {
+        slug: "test-rollout",
+        preferences: [{ preferenceName: "test.pref", value: 1 }],
+      },
+    };
+
+    const action = new PreferenceRolloutAction();
+    await action.processRecipe(recipe, BaseAction.suitability.FILTER_MATCH);
+    await action.finalize();
+    is(action.lastError, null, "lastError should be null");
+
+    is(Services.prefs.getIntPref("test.pref"), 1, "pref should not change");
+
+    // start up pref isn't set
+    is(
+      Services.prefs.getPrefType("app.normandy.startupRolloutPrefs.test.pref"),
+      Services.prefs.PREF_INVALID,
+      "startup pref1 should not be set"
+    );
+
+    // rollout was not stored
+    Assert.deepEqual(
+      await PreferenceRollouts.getAll(),
+      [],
+      "Rollout should not be stored in db"
+    );
+
+    sendEventStub.assertEvents([]);
+    Assert.deepEqual(
+      setExperimentActiveStub.args,
+      [],
+      "a telemetry experiment should not be activated"
+    );
+
+    // Cleanup
+    Services.prefs.getDefaultBranch("").deleteBranch("test.pref");
+  }
+);
--- a/toolkit/components/normandy/test/browser/head.js
+++ b/toolkit/components/normandy/test/browser/head.js
@ -325,10 +325,10 @@ this.studyEndObserved = function(recipeId) {
  );
 };

-this.withSendEventStub = function(testFunction) {
+this.withSendEventSpy = function(testFunction) {
  return async function wrappedTestFunction(...args) {
-    const stub = sinon.spy(TelemetryEvents, "sendEvent");
-    stub.assertEvents = expected => {
+    const spy = sinon.spy(TelemetryEvents, "sendEvent");
+    spy.assertEvents = expected => {
      expected = expected.map(event => ["normandy"].concat(event));
      TelemetryTestUtils.assertEvents(
        expected,
@ -338,10 +338,10 @@ this.withSendEventStub = function(testFunction) {
    };
    Services.telemetry.clearEvents();
    try {
-      await testFunction(...args, stub);
+      await testFunction(...args, spy);
    } finally {
-      stub.restore();
-      Assert.ok(!stub.threw(), "Telemetry events should not fail");
+      spy.restore();
+      Assert.ok(!spy.threw(), "Telemetry events should not fail");
    }
  };
 };
--- a/toolkit/components/telemetry/Events.yaml
+++ b/toolkit/components/telemetry/Events.yaml
@ -702,6 +702,7 @@ normandy:
    expiry_version: never
    extra_keys:
      enrollmentId: A unique ID for this enrollment that will be included in all related Telemetry.
+      reason: The reason the rollout graduated

  expose:
    objects: [