test(cirrus): Cirrus preview flag (#11167)

Because

- We added preview flag support for Cirrus to enable them to QA

This commit

- Add a new test to test preview flag functionality

Fixes #11103 
Note: Working on to send it to main collection

.env.integration-tests

@@ -48,7 +48,7 @@ STATSD_PREFIX=experimenter
 UPLOADS_FILE_STORAGE=django.core.files.storage.FileSystemStorage
 CIRRUS_REMOTE_SETTING_REFRESH_RATE_IN_SECONDS=1
 CIRRUS_REMOTE_SETTING_URL=http://kinto:8888/v1/buckets/main/collections/nimbus-web-experiments/records
-CIRRUS_REMOTE_SETTING_PREVIEW_URL=
+CIRRUS_REMOTE_SETTING_PREVIEW_URL=http://kinto:8888/v1/buckets/main-workspace/collections/nimbus-web-preview/records
 CIRRUS_APP_ID=demo-app-beta
 CIRRUS_APP_NAME=demo_app
 CIRRUS_CHANNEL=release

.env.sample

@@ -52,7 +52,7 @@ UPLOADS_GS_BUCKET_NAME=
 UPLOADS_FILE_STORAGE=django.core.files.storage.FileSystemStorage
 CIRRUS_REMOTE_SETTING_REFRESH_RATE_IN_SECONDS=10
 CIRRUS_REMOTE_SETTING_URL=http://kinto:8888/v1/buckets/main/collections/nimbus-web-experiments/records
-CIRRUS_REMOTE_SETTING_PREVIEW_URL=http://kinto:8888/v1/buckets/main/collections/nimbus-web-preview/records
+CIRRUS_REMOTE_SETTING_PREVIEW_URL=http://kinto:8888/v1/buckets/main-workspace/collections/nimbus-web-preview/records
 CIRRUS_APP_ID=demo-app-beta
 CIRRUS_APP_NAME=demo_app
 CIRRUS_CHANNEL=beta

.env.test

@@ -1,5 +1,5 @@
 CIRRUS_REMOTE_SETTING_URL=http://kinto:8888/v1/buckets/main/collections/nimbus-web-experiments/records
-CIRRUS_REMOTE_SETTING_PREVIEW_URL=http://kinto:8888/v1/buckets/main/collections/nimbus-web-preview/records
+CIRRUS_REMOTE_SETTING_PREVIEW_URL=http://kinto:8888/v1/buckets/main-workspace/collections/nimbus-web-preview/records
 CIRRUS_REMOTE_SETTING_REFRESH_RATE_IN_SECONDS=10
 CIRRUS_APP_ID=test_app_id
 CIRRUS_APP_NAME=test_app_name

demo-app/frontend/src/index.js

@@ -13,10 +13,11 @@ function App() {
   const [clientId, setClientId] = useState('');
   const [context, setContext] = useState('');
   const [apiCallTriggered, setApiCallTriggered] = useState(false);
+  const [nimbusPreview, setNimbusPreview] = useState(false);
 
   useEffect(() => {
     if (apiCallTriggered) {
-      const apiUrl = '/api/data';
+      const apiUrl = `/api/data?nimbus_preview=${nimbusPreview}`;
 
       fetch(apiUrl, {
         method: 'GET',
@@ -35,7 +36,7 @@ function App() {
           setApiCallTriggered(false);
         });
     }
-  }, [apiCallTriggered, clientId, context]);
+  }, [apiCallTriggered, clientId, context, nimbusPreview]);
 
   const displayText = message && message['example-feature'] && message['example-feature']['something']
     ? message['example-feature']['something']
@@ -57,9 +58,16 @@ function App() {
           value={context}
           onChange={(e) => setContext(e.target.value)}
         />
+        <label>
+          <input
+            type="checkbox"
+            checked={nimbusPreview}
+            onChange={(e) => setNimbusPreview(e.target.checked)}
+          />
+          Nimbus Preview
+        </label>
         <button onClick={() => setApiCallTriggered(true)}>Send My Details</button>
       </div>
     </div>
   );
 }
-

demo-app/server/index.js

@@ -1,7 +1,10 @@
 const http = require('http');
+const url = require('url');
 
 const server = http.createServer(async (req, res) => {
-  if (req.url === '/api/data' && req.method === 'GET') {
+  const parsedUrl = url.parse(req.url, true);
+
+  if (parsedUrl.pathname === '/api/data' && req.method === 'GET') {
     res.setHeader('Content-Type', 'application/json');
 
     const defaultApiInput = {
@@ -14,21 +17,26 @@ const server = http.createServer(async (req, res) => {
         },
       },
     };
-
+    // Get client ID and context and nimbus preview flag from request headers
-    // Get client ID and context from request headers
     const clientID = req.headers['x-client-id'] || defaultApiInput.client_id;
     const contextJSON = req.headers['x-context'] || JSON.stringify(defaultApiInput.context);
     const context = JSON.parse(contextJSON);
+    const nimbusPreview = parsedUrl.query.nimbus_preview === 'true';
 
     const apiInput = {
       client_id: clientID,
       context: context,
+      nimbus_preview: nimbusPreview,
     };
 
+    let pathName = '/v1/features/'
+    if(nimbusPreview) {
+      pathName= pathName+"?nimbus_preview=" + nimbusPreview;
+    }
     const options = {
-      hostname: 'cirrus', // Use the service name
+      hostname: 'cirrus',
       port: 8001,
-      path: '/v1/features/',
+      path: pathName,
       method: 'POST',
       headers: {
         'Content-Type': 'application/json',

experimenter/tests/integration/nimbus/pages/demo_app/frontend.py

@@ -7,31 +7,74 @@ from nimbus.pages.base import Base
 
 class DemoAppPage(Base):
     PAGE_TITLE = "Demo app frontend"
+    CONTROL_BRANCH = "Control branch"
+    TREATMENT_BRANCH = "Treatment branch"
+
+    # XPaths for elements
+    CLIENT_ID_INPUT_XPATH = "//input[@placeholder='Client ID']"
+    CONTEXT_INPUT_XPATH = "//input[@placeholder='Context']"
+    PREVIEW_CHECKBOX_XPATH = "//input[@type='checkbox']"
+    SEND_DETAILS_BUTTON_XPATH = "//button[contains(text(), 'Send My Details')]"
+    RESULT_TEXT_XPATH_TEMPLATE = "//h1[{}]"
+
+    # Client dictionaries
+    CLIENT_1 = {"client_id": "dummy_client_id", "context": '{"test1":"test2"}'}
+
+    CLIENT_2 = {"client_id": "example1", "context": '{"test1":"test2"}'}
+
+    CLIENT_3 = {"client_id": "test", "context": '{"language":"en", "region":"CA"}'}
+
+    # Branch value constants
+    REFERENCE_BRANCH_VALUE = f'{{"enabled": true, "something": "{CONTROL_BRANCH}"}}'
+    TREATMENT_BRANCH_VALUE = f'{{"enabled": true, "something": "{TREATMENT_BRANCH}"}}'
+
+    # Default values
+    DEFAULT_NOT_ENROLLED_TEXT = "Not Enrolled"
+    DEFAULT_WICKED_TEXT = "wicked"
 
     def wait_for_result_text(self, text_list):
-        xpath_conditions = " | ".join([f".='{text}'" for text in text_list])
+        """Wait for and return the result text element matching any text in text_list."""
-        xpath = f"//h1[{' or '.join(xpath_conditions.split(' | '))}]"
+        xpath_conditions = " or ".join([f".='{text}'" for text in text_list])
+        xpath = self.RESULT_TEXT_XPATH_TEMPLATE.format(xpath_conditions)
+        return self.wait_for_and_find_element(By.XPATH, xpath, description="Result Text")
 
-        return self.wait_for_and_find_element(By.XPATH, xpath, description=None)
+    def fill_form(self, client_id, context):
+        """Fill the form with client_id and context."""
+        self.enter_text(self.CLIENT_ID_INPUT_XPATH, client_id, "Client ID Input")
+        self.enter_text(self.CONTEXT_INPUT_XPATH, context, "Context Input")
 
-    def fill_and_send_form_data(self, client_id, context):
+    def enable_nimbus_preview(self):
-        client_xpath = "//input[@placeholder='Client ID']"
+        """Enable Nimbus Preview checkbox if not already selected."""
-        client_id_input = self.wait_for_and_find_element(
+        preview_checkbox = self.wait_for_and_find_element(
-            By.XPATH, client_xpath, description=None
+            By.XPATH, self.PREVIEW_CHECKBOX_XPATH, description="Nimbus Preview Checkbox"
         )
-        context_xpath = "//input[@placeholder='Context']"
+        if not preview_checkbox.is_selected():
-        context_input = self.wait_for_and_find_element(
+            preview_checkbox.click()
-            By.XPATH, context_xpath, description=None
-        )
-        time.sleep(10)
-
-        client_id_input.send_keys(client_id)
-        context_input.send_keys(context)
 
     def click_send_my_details(self):
-        details_xpath = "//button[contains(text(), 'Send My Details')]"
+        """Click the 'Send My Details' button."""
-        send_details_button = self.wait_for_and_find_element(
+        self.click_button(self.SEND_DETAILS_BUTTON_XPATH, "Send My Details Button")
-            By.XPATH, details_xpath, description=None
-        )
 
-        send_details_button.click()
+    def fill_and_send_form_data(self, client_id, context, nimbus_preview=False):
+        """Fill the form with client_id and context, \
+            optionally enable Nimbus preview, and submit the form."""
+        self.fill_form(client_id, context)
+        if nimbus_preview:
+            self.enable_nimbus_preview()
+        time.sleep(10)
+        self.click_send_my_details()
+
+    def enter_text(self, xpath, text, description):
+        """Helper method to enter text into an input field."""
+        input_element = self.wait_for_and_find_element(
+            By.XPATH, xpath, description=description
+        )
+        input_element.clear()
+        input_element.send_keys(text)
+
+    def click_button(self, xpath, description):
+        """Helper method to click a button."""
+        button_element = self.wait_for_and_find_element(
+            By.XPATH, xpath, description=description
+        )
+        button_element.click()

experimenter/tests/integration/nimbus/test_cirrus_integration.py

@@ -20,9 +20,10 @@ def test_create_new_rollout_approve_remote_settings_cirrus(
     demo_app,
 ):
     # Launch a rollout with 100% population
-    reference_branch_value = '{"enabled": true, "something": "You are enrolled"}'
     create_experiment(
-        selenium, is_rollout=True, reference_branch_value=reference_branch_value
+        selenium,
+        is_rollout=True,
+        reference_branch_value=demo_app.REFERENCE_BRANCH_VALUE,
     ).launch_and_approve()
 
     kinto_client.approve()
@@ -31,20 +32,20 @@ def test_create_new_rollout_approve_remote_settings_cirrus(
 
     HomePage(selenium, base_url).open().find_in_table(experiment_name)
 
-    # demo app frontend, default displays "Not Enrolled"
+    # Demo app frontend, default displays "Not Enrolled"
     navigate_to(selenium)
     selenium.refresh()
-    result_text_element = demo_app.wait_for_result_text(["Not Enrolled"])
+    assert demo_app.wait_for_result_text(
-    assert result_text_element.is_displayed()
+        [demo_app.DEFAULT_NOT_ENROLLED_TEXT]
+    ).is_displayed()
 
-    # send client_id and context in a request to backend, backend will connect to
+    # Send client_id and context to backend
-    # cirrus and will return back the response
+    demo_app.fill_and_send_form_data(
-    demo_app.fill_and_send_form_data("dummy_client_id", '{"test1":"test2"}')
+        demo_app.CLIENT_1["client_id"], demo_app.CLIENT_1["context"]
-    demo_app.click_send_my_details()
+    )
 
-    # it should render "You are enrolled", reference branch value
+    # Should render "You are enrolled"
-    result_text_element = demo_app.wait_for_result_text(["You are enrolled"])
+    assert demo_app.wait_for_result_text([demo_app.CONTROL_BRANCH]).is_displayed()
-    assert result_text_element.is_displayed()
 
     # Unenroll
     summary = SummaryPage(selenium, experiment_url).open()
@@ -56,17 +57,17 @@ def test_create_new_rollout_approve_remote_settings_cirrus(
     # Demo app frontend, default displays "Not Enrolled" again
     navigate_to(selenium)
     selenium.refresh()
-
+    assert demo_app.wait_for_result_text(
-    result_text_element = demo_app.wait_for_result_text(["Not Enrolled"])
+        [demo_app.DEFAULT_NOT_ENROLLED_TEXT]
-    assert result_text_element.is_displayed()
+    ).is_displayed()
 
     # Send the same client_id and context after the rollout has ended
-    demo_app.fill_and_send_form_data("dummy_client_id", '{"test1":"test2"}')
+    demo_app.fill_and_send_form_data(
-    demo_app.click_send_my_details()
+        demo_app.CLIENT_1["client_id"], demo_app.CLIENT_1["context"]
+    )
 
-    # returns the default value
+    # Returns the default value
-    result_text_element = demo_app.wait_for_result_text(["wicked"])
+    assert demo_app.wait_for_result_text([demo_app.DEFAULT_WICKED_TEXT]).is_displayed()
-    assert result_text_element.is_displayed()
 
 
 @pytest.mark.cirrus_enrollment
@@ -80,12 +81,10 @@ def test_create_new_experiment_approve_remote_settings_cirrus(
     demo_app,
 ):
     # Launch an experiment with two branches
-    reference_branch_value = '{"enabled": true, "something": "Control branch"}'
-    treatment_branch_value = '{"enabled": true, "something": "Treatment branch"}'
     create_experiment(
         selenium,
-        reference_branch_value=reference_branch_value,
+        reference_branch_value=demo_app.REFERENCE_BRANCH_VALUE,
-        treatment_branch_value=treatment_branch_value,
+        treatment_branch_value=demo_app.TREATMENT_BRANCH_VALUE,
     ).launch_and_approve()
 
     kinto_client.approve()
@@ -98,33 +97,36 @@ def test_create_new_experiment_approve_remote_settings_cirrus(
     navigate_to(selenium)
     selenium.refresh()
 
-    result_text_element = demo_app.wait_for_result_text(["Not Enrolled"])
+    assert demo_app.wait_for_result_text(
-    assert result_text_element.is_displayed()
+        [demo_app.DEFAULT_NOT_ENROLLED_TEXT]
+    ).is_displayed()
 
     # Pass client_id and context
-    demo_app.fill_and_send_form_data("test", '{"test1":"test2"}')
+    demo_app.fill_and_send_form_data(
-    demo_app.click_send_my_details()
+        demo_app.CLIENT_1["client_id"], demo_app.CLIENT_1["context"]
+    )
 
     # Determine the variation displayed and assert accordingly
-    # Check if either "Control branch" or "Treatment branch" is displayed
     displayed_text = demo_app.wait_for_result_text(
-        ["Control branch", "Treatment branch"]
+        [demo_app.CONTROL_BRANCH, demo_app.TREATMENT_BRANCH]
     ).text
-    assert displayed_text in ["Control branch", "Treatment branch"]
+    assert displayed_text in [demo_app.CONTROL_BRANCH, demo_app.TREATMENT_BRANCH]
 
     # Refresh the page and try passing a new client
     selenium.refresh()
-    result_text_element = demo_app.wait_for_result_text(["Not Enrolled"])
+    assert demo_app.wait_for_result_text(
-    assert result_text_element.is_displayed()
+        [demo_app.DEFAULT_NOT_ENROLLED_TEXT]
+    ).is_displayed()
 
-    demo_app.fill_and_send_form_data("example1", '{"test1":"test2"}')
+    demo_app.fill_and_send_form_data(
-    demo_app.click_send_my_details()
+        demo_app.CLIENT_2["client_id"], demo_app.CLIENT_2["context"]
+    )
 
     # Determine the variation displayed and assert accordingly
     displayed_text = demo_app.wait_for_result_text(
-        ["Control branch", "Treatment branch"]
+        [demo_app.CONTROL_BRANCH, demo_app.TREATMENT_BRANCH]
     ).text
-    assert displayed_text in ["Control branch", "Treatment branch"]
+    assert displayed_text in [demo_app.CONTROL_BRANCH, demo_app.TREATMENT_BRANCH]
 
     # Unenroll
     summary = SummaryPage(selenium, experiment_url).open()
@@ -136,26 +138,26 @@ def test_create_new_experiment_approve_remote_settings_cirrus(
     navigate_to(selenium)
     selenium.refresh()
 
-    result_text_element = demo_app.wait_for_result_text(["Not Enrolled"])
+    assert demo_app.wait_for_result_text(
-    assert result_text_element.is_displayed()
+        [demo_app.DEFAULT_NOT_ENROLLED_TEXT]
+    ).is_displayed()
 
     # Send the same client_id and context after the experiment has ended
-    demo_app.fill_and_send_form_data("example1", '{"test1":"test2"}')
+    demo_app.fill_and_send_form_data(
-    demo_app.click_send_my_details()
+        demo_app.CLIENT_2["client_id"], demo_app.CLIENT_2["context"]
+    )
 
-    # returns the default value
+    # Returns the default value
-    result_text_element = demo_app.wait_for_result_text(["wicked"])
+    assert demo_app.wait_for_result_text([demo_app.DEFAULT_WICKED_TEXT]).is_displayed()
-    assert result_text_element.is_displayed()
 
     # Check another client id
     selenium.refresh()
-    # Send the same client_id and context after the experiment has ended
+    demo_app.fill_and_send_form_data(
-    demo_app.fill_and_send_form_data("test", '{"test1":"test2"}')
+        demo_app.CLIENT_1["client_id"], demo_app.CLIENT_1["context"]
-    demo_app.click_send_my_details()
+    )
 
-    # returns the default value
+    # Returns the default value
-    result_text_element = demo_app.wait_for_result_text(["wicked"])
+    assert demo_app.wait_for_result_text([demo_app.DEFAULT_WICKED_TEXT]).is_displayed()
-    assert result_text_element.is_displayed()
 
 
 @pytest.mark.cirrus_enrollment
@@ -169,12 +171,10 @@ def test_check_cirrus_targeting(
     demo_app,
 ):
     # Launch an experiment with two branches
-    reference_branch_value = '{"enabled": true, "something": "Control branch"}'
-    treatment_branch_value = '{"enabled": true, "something": "Treatment branch"}'
     create_experiment(
         selenium,
-        reference_branch_value=reference_branch_value,
+        reference_branch_value=demo_app.REFERENCE_BRANCH_VALUE,
-        treatment_branch_value=treatment_branch_value,
+        treatment_branch_value=demo_app.TREATMENT_BRANCH_VALUE,
         languages=True,
         countries=True,
     ).launch_and_approve()
@@ -189,33 +189,36 @@ def test_check_cirrus_targeting(
     navigate_to(selenium)
     selenium.refresh()
 
-    result_text_element = demo_app.wait_for_result_text(["Not Enrolled"])
+    assert demo_app.wait_for_result_text(
-    assert result_text_element.is_displayed()
+        [demo_app.DEFAULT_NOT_ENROLLED_TEXT]
+    ).is_displayed()
 
     # Pass client_id and context
-    demo_app.fill_and_send_form_data("test", '{"language":"en", "region":"CA"}')
+    demo_app.fill_and_send_form_data(
-    demo_app.click_send_my_details()
+        demo_app.CLIENT_3["client_id"], demo_app.CLIENT_3["context"]
+    )
 
     # Determine the variation displayed and assert accordingly
-    # Check if either "Control branch" or "Treatment branch" is displayed
     displayed_text = demo_app.wait_for_result_text(
-        ["Control branch", "Treatment branch"]
+        [demo_app.CONTROL_BRANCH, demo_app.TREATMENT_BRANCH]
     ).text
-    assert displayed_text in ["Control branch", "Treatment branch"]
+    assert displayed_text in [demo_app.CONTROL_BRANCH, demo_app.TREATMENT_BRANCH]
 
     # Refresh the page and try passing a new client
     selenium.refresh()
-    result_text_element = demo_app.wait_for_result_text(["Not Enrolled"])
+    assert demo_app.wait_for_result_text(
-    assert result_text_element.is_displayed()
+        [demo_app.DEFAULT_NOT_ENROLLED_TEXT]
+    ).is_displayed()
 
-    demo_app.fill_and_send_form_data("example1", '{"language":"en", "region":"CA"}')
+    demo_app.fill_and_send_form_data(
-    demo_app.click_send_my_details()
+        demo_app.CLIENT_2["client_id"], demo_app.CLIENT_3["context"]
+    )
 
     # Determine the variation displayed and assert accordingly
     displayed_text = demo_app.wait_for_result_text(
-        ["Control branch", "Treatment branch"]
+        [demo_app.CONTROL_BRANCH, demo_app.TREATMENT_BRANCH]
     ).text
-    assert displayed_text in ["Control branch", "Treatment branch"]
+    assert displayed_text in [demo_app.CONTROL_BRANCH, demo_app.TREATMENT_BRANCH]
 
     # Unenroll
     summary = SummaryPage(selenium, experiment_url).open()
@@ -227,23 +230,73 @@ def test_check_cirrus_targeting(
     navigate_to(selenium)
     selenium.refresh()
 
-    result_text_element = demo_app.wait_for_result_text(["Not Enrolled"])
+    assert demo_app.wait_for_result_text(
-    assert result_text_element.is_displayed()
+        [demo_app.DEFAULT_NOT_ENROLLED_TEXT]
+    ).is_displayed()
 
     # Send the same client_id and context after the experiment has ended
-    demo_app.fill_and_send_form_data("example1", '{"language":"en", "region":"CA"}')
+    demo_app.fill_and_send_form_data(
-    demo_app.click_send_my_details()
+        demo_app.CLIENT_3["client_id"], demo_app.CLIENT_3["context"]
+    )
 
-    # returns the default value
+    # Returns the default value
-    result_text_element = demo_app.wait_for_result_text(["wicked"])
+    assert demo_app.wait_for_result_text([demo_app.DEFAULT_WICKED_TEXT]).is_displayed()
-    assert result_text_element.is_displayed()
 
     # Check another client id
     selenium.refresh()
-    # Send the same client_id and context after the experiment has ended
+    demo_app.fill_and_send_form_data(
-    demo_app.fill_and_send_form_data("test", '{"language":"en", "region":"CA"}')
+        demo_app.CLIENT_1["client_id"], demo_app.CLIENT_3["context"]
-    demo_app.click_send_my_details()
+    )
 
-    # returns the default value
+    # Returns the default value
-    result_text_element = demo_app.wait_for_result_text(["wicked"])
+    assert demo_app.wait_for_result_text([demo_app.DEFAULT_WICKED_TEXT]).is_displayed()
-    assert result_text_element.is_displayed()
+
+
+@pytest.mark.cirrus_enrollment
+def test_nimbus_preview_flag(
+    selenium,
+    experiment_url,
+    create_experiment,
+    kinto_client,
+    base_url,
+    experiment_name,
+    demo_app,
+):
+    create_experiment(
+        selenium,
+        reference_branch_value=demo_app.REFERENCE_BRANCH_VALUE,
+        treatment_branch_value=demo_app.TREATMENT_BRANCH_VALUE,
+    ).launch_to_preview()
+
+    SummaryPage(selenium, experiment_url).open().wait_for_preview_status()
+
+    navigate_to(selenium)
+    selenium.refresh()
+    assert demo_app.wait_for_result_text(
+        [demo_app.DEFAULT_NOT_ENROLLED_TEXT]
+    ).is_displayed()
+
+    # Enable nimbus preview flag
+    demo_app.fill_and_send_form_data(
+        demo_app.CLIENT_1["client_id"],
+        demo_app.CLIENT_1["context"],
+        nimbus_preview=True,
+    )
+
+    displayed_text = demo_app.wait_for_result_text(
+        [demo_app.CONTROL_BRANCH, demo_app.TREATMENT_BRANCH]
+    ).text
+    assert displayed_text in [demo_app.CONTROL_BRANCH, demo_app.TREATMENT_BRANCH]
+
+    navigate_to(selenium)
+    selenium.refresh()
+    assert demo_app.wait_for_result_text(
+        [demo_app.DEFAULT_NOT_ENROLLED_TEXT]
+    ).is_displayed()
+
+    # Not using nimbus preview flag
+    demo_app.fill_and_send_form_data(
+        demo_app.CLIENT_2["client_id"], demo_app.CLIENT_2["context"]
+    )
+
+    assert demo_app.wait_for_result_text([demo_app.DEFAULT_WICKED_TEXT]).is_displayed()