Fixes #1106 - Add high population marker to preference experiments.

recipe-client-addon/lib/PreferenceExperiments.jsm

@@ -46,6 +46,8 @@
  *   preference is modified on startup of the add-on. If "user", the user value
  *   for the preference is modified when the experiment starts, and is reset to
  *   its original value when the experiment ends.
+ * @property {string} experimentType
+ *   The type to report to Telemetry's experiment marker API.
  */
 
 "use strict";
@@ -188,7 +190,7 @@ this.PreferenceExperiments = {
       TelemetryEnvironment.setExperimentActive(
         experiment.name,
         experiment.branch,
-        {type: "normandy-preference-experiment"}
+        {type: experiment.experimentType}
       );
 
       // Watch for changes to the experiment's preference
@@ -273,7 +275,15 @@ this.PreferenceExperiments = {
    *   - if an experiment for the given preference is active
    *   - If the given preferenceType does not match the existing stored preference
    */
-  async start({name, branch, preferenceName, preferenceValue, preferenceBranchType, preferenceType}) {
+  async start({
+    name,
+    branch,
+    preferenceName,
+    preferenceValue,
+    preferenceBranchType,
+    preferenceType,
+    experimentType = "normandy-pref",
+  }) {
     log.debug(`PreferenceExperiments.start(${name}, ${branch})`);
 
     const store = await ensureStorage();
@@ -296,6 +306,12 @@ this.PreferenceExperiments = {
       throw new Error(`Invalid value for preferenceBranchType: ${preferenceBranchType}`);
     }
 
+    if (!experimentType.startsWith("normandy-pref")) {
+      throw new Error(
+        `Expected experimentType to begin with "normandy-pref". Found "${experimentType}".`
+      );
+    }
+
     /** @type {Experiment} */
     const experiment = {
       name,
@@ -307,6 +323,7 @@ this.PreferenceExperiments = {
       preferenceType,
       previousPreferenceValue: getPref(preferences, preferenceName, preferenceType),
       preferenceBranchType,
+      experimentType,
     };
 
     const prevPrefType = Services.prefs.getPrefType(preferenceName);
@@ -328,7 +345,7 @@ this.PreferenceExperiments = {
     store.data[name] = experiment;
     store.saveSoon();
 
-    TelemetryEnvironment.setExperimentActive(name, branch, {type: "normandy-preference-experiment"});
+    TelemetryEnvironment.setExperimentActive(name, branch, {type: experimentType});
     await this.saveStartupPrefs();
   },
 

recipe-client-addon/test/browser/browser_PreferenceExperiments.js

@@ -16,10 +16,11 @@ function experimentFactory(attrs) {
     expired: false,
     lastSeen: new Date().toJSON(),
     preferenceName: "fake.preference",
-    preferenceValue: "falkevalue",
+    preferenceValue: "fakevalue",
     preferenceType: "string",
     previousPreferenceValue: "oldfakevalue",
     preferenceBranchType: "default",
+    experimentType: "normandy-pref",
   }, attrs);
 }
 
@@ -561,12 +562,37 @@ decorate_task(
 
     await PreferenceExperiments.init();
     ok(
-      setActiveStub.calledWith("test", "branch", {type: "normandy-preference-experiment"}),
+      setActiveStub.calledWith("test", "branch", {type: "normandy-pref"}),
       "Experiment is registered by init",
     );
   },
 );
 
+// init should use the provided experiment type
+decorate_task(
+  withMockExperiments,
+  withMockPreferences,
+  withStub(TelemetryEnvironment, "setExperimentActive"),
+  withStub(PreferenceExperiments, "startObserver"),
+  async function testInit(experiments, mockPreferences, setActiveStub, startObserverStub) {
+    mockPreferences.set("fake.pref", "experiment value");
+
+    experiments.test = experimentFactory({
+      name: "test",
+      branch: "branch",
+      preferenceName: "fake.pref",
+      preferenceValue: "experiment value",
+      experimentType: "normandy-pref-test",
+    });
+
+    await PreferenceExperiments.init();
+    ok(
+      setActiveStub.calledWith("test", "branch", {type: "normandy-pref-test"}),
+      "init should use the provided experiment type",
+    );
+  },
+);
+
 // starting and stopping experiments should register in telemetry
 decorate_task(
   withMockExperiments,
@@ -583,7 +609,7 @@ decorate_task(
     });
 
     ok(
-      setActiveStub.calledWith("test", "branch", {type: "normandy-preference-experiment"}),
+      setActiveStub.calledWith("test", "branch", {type: "normandy-pref"}),
       "Experiment is registerd by start()",
     );
     await PreferenceExperiments.stop("test");
@@ -591,6 +617,30 @@ decorate_task(
   },
 );
 
+// starting experiments should use the provided experiment type
+decorate_task(
+  withMockExperiments,
+  withStub(TelemetryEnvironment, "setExperimentActive"),
+  withStub(TelemetryEnvironment, "setExperimentInactive"),
+  async function testInitTelemetry(experiments, setActiveStub, setInactiveStub) {
+    await PreferenceExperiments.start({
+      name: "test",
+      branch: "branch",
+      preferenceName: "fake.preference",
+      preferenceValue: "value",
+      preferenceType: "string",
+      preferenceBranchType: "default",
+      experimentType: "normandy-pref-test",
+    });
+
+    ok(
+      setActiveStub.calledWith("test", "branch", {type: "normandy-pref-test"}),
+      "start() should register the experiment with the provided type",
+    );
+  },
+);
+
+
 // Experiments shouldn't be recorded by init() in telemetry if they are expired
 decorate_task(
   withMockExperiments,

recipe-server/client/actions/preference-experiment/index.js

@@ -19,7 +19,12 @@ export function resetAction() {
 export default class PreferenceExperimentAction extends Action {
   async execute() {
     const {
-      slug, preferenceName, preferenceBranchType, branches, preferenceType,
+      branches,
+      isHighPopulation,
+      preferenceBranchType,
+      preferenceName,
+      preferenceType,
+      slug,
     } = this.recipe.arguments;
     const experiments = this.normandy.preferenceExperiments;
 
@@ -54,6 +59,7 @@ export default class PreferenceExperimentAction extends Action {
 
       // Otherwise, enroll!
       const branch = await this.chooseBranch(branches);
+      const experimentType = isHighPopulation ? 'normandy-exp-highpop' : 'normandy-exp';
       await experiments.start({
         name: slug,
         branch: branch.slug,
@@ -61,6 +67,7 @@ export default class PreferenceExperimentAction extends Action {
         preferenceValue: branch.value,
         preferenceBranchType,
         preferenceType,
+        experimentType,
       });
     } else {
       // If the experiment exists, and isn't expired, bump the lastSeen date.

recipe-server/client/actions/preference-experiment/package.json

@@ -44,6 +44,11 @@
                     "enum": ["user", "default"],
                     "default": "default"
                 },
+                "isHighPopulation": {
+                    "description": "Marks the preference experiment as a high population experiment, that should be excluded from certain types of telemetry",
+                    "type": "boolean",
+                    "default": "false"
+                },
                 "branches": {
                     "description": "List of experimental branches",
                     "type": "array",

recipe-server/client/actions/tests/test_preference-experiment.js

@@ -184,6 +184,7 @@ describe('PreferenceExperimentAction', () => {
           preferenceValue: 'branch1',
           preferenceBranchType: 'user',
           preferenceType: 'string',
+          experimentType: 'normandy-exp',
         });
     });
 

recipe-server/client/control/components/recipes/PreferenceExperimentFields.js

@@ -1,5 +1,5 @@
 /* eslint-disable react/jsx-boolean-value */
-import { Row, Col, Alert, Button, Icon, Input, InputNumber, Radio, Select } from 'antd';
+import { Row, Col, Alert, Button, Checkbox, Icon, Input, InputNumber, Radio, Select } from 'antd';
 import autobind from 'autobind-decorator';
 import { List, Map } from 'immutable';
 import PropTypes from 'prop-types';
@@ -49,6 +49,14 @@ export default class PreferenceExperimentFields extends React.Component {
           >
             <DocumentUrlInput disabled={disabled} />
           </FormItem>
+
+          <FormItem
+            label="High volume recipe?"
+            name="arguments.isHighVolume"
+            initialValue={recipeArguments.get('isHighVolume')}
+          >
+            <Checkbox />
+          </FormItem>
         </Col>
 
         <Col sm={24} md={{ span: 12, offset: 1 }}>