mozilla
/
experimenter
зеркало из https://github.com/mozilla/experimenter.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

test(cirrus): Cirrus preview flag (#11167) 

Because

- We added preview flag support for Cirrus to enable them to QA

This commit

- Add a new test to test preview flag functionality

Fixes #11103 
Note: Working on to send it to main collection
This commit is contained in:
Yashika Khurana 2024-08-26 08:56:27 -07:00 коммит произвёл GitHub
Родитель d977e025ab
Коммит 9320b4c603
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: B5690EEEBB952194
7 изменённых файлов: 223 добавлений и 111 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

2
.env.integration-tests
Просмотреть файл

@ -48,7 +48,7 @@ STATSD_PREFIX=experimenter
UPLOADS_FILE_STORAGE=django.core.files.storage.FileSystemStorage
CIRRUS_REMOTE_SETTING_REFRESH_RATE_IN_SECONDS=1
CIRRUS_REMOTE_SETTING_URL=http://kinto:8888/v1/buckets/main/collections/nimbus-web-experiments/records
CIRRUS_REMOTE_SETTING_PREVIEW_URL=
CIRRUS_REMOTE_SETTING_PREVIEW_URL=http://kinto:8888/v1/buckets/main-workspace/collections/nimbus-web-preview/records
CIRRUS_APP_ID=demo-app-beta
CIRRUS_APP_NAME=demo_app
CIRRUS_CHANNEL=release

2
.env.sample
Просмотреть файл

@ -52,7 +52,7 @@ UPLOADS_GS_BUCKET_NAME=
UPLOADS_FILE_STORAGE=django.core.files.storage.FileSystemStorage
CIRRUS_REMOTE_SETTING_REFRESH_RATE_IN_SECONDS=10
CIRRUS_REMOTE_SETTING_URL=http://kinto:8888/v1/buckets/main/collections/nimbus-web-experiments/records
CIRRUS_REMOTE_SETTING_PREVIEW_URL=http://kinto:8888/v1/buckets/main/collections/nimbus-web-preview/records
CIRRUS_REMOTE_SETTING_PREVIEW_URL=http://kinto:8888/v1/buckets/main-workspace/collections/nimbus-web-preview/records
CIRRUS_APP_ID=demo-app-beta
CIRRUS_APP_NAME=demo_app
CIRRUS_CHANNEL=beta

2
.env.test
Просмотреть файл

@ -1,5 +1,5 @@
CIRRUS_REMOTE_SETTING_URL=http://kinto:8888/v1/buckets/main/collections/nimbus-web-experiments/records
CIRRUS_REMOTE_SETTING_PREVIEW_URL=http://kinto:8888/v1/buckets/main/collections/nimbus-web-preview/records
CIRRUS_REMOTE_SETTING_PREVIEW_URL=http://kinto:8888/v1/buckets/main-workspace/collections/nimbus-web-preview/records
CIRRUS_REMOTE_SETTING_REFRESH_RATE_IN_SECONDS=10
CIRRUS_APP_ID=test_app_id
CIRRUS_APP_NAME=test_app_name

14
demo-app/frontend/src/index.js
Просмотреть файл

@ -13,10 +13,11 @@ function App() {
const [clientId, setClientId] = useState('');
const [context, setContext] = useState('');
const [apiCallTriggered, setApiCallTriggered] = useState(false);
const [nimbusPreview, setNimbusPreview] = useState(false);
useEffect(() => {
if (apiCallTriggered) {
const apiUrl = '/api/data';
const apiUrl = `/api/data?nimbus_preview=${nimbusPreview}`;
fetch(apiUrl, {
method: 'GET',
@ -35,7 +36,7 @@ function App() {
setApiCallTriggered(false);
});
}
}, [apiCallTriggered, clientId, context]);
}, [apiCallTriggered, clientId, context, nimbusPreview]);
const displayText = message && message['example-feature'] && message['example-feature']['something']
? message['example-feature']['something']
@ -57,9 +58,16 @@ function App() {
value={context}
onChange={(e) => setContext(e.target.value)}
/>
<label>
<input
type="checkbox"
checked={nimbusPreview}
onChange={(e) => setNimbusPreview(e.target.checked)}
/>
Nimbus Preview
</label>
<button onClick={() => setApiCallTriggered(true)}>Send My Details</button>
</div>
</div>
);
}

18
demo-app/server/index.js
Просмотреть файл

@ -1,7 +1,10 @@
const http = require('http');
const url = require('url');
const server = http.createServer(async (req, res) => {
if (req.url === '/api/data' && req.method === 'GET') {
const parsedUrl = url.parse(req.url, true);
if (parsedUrl.pathname === '/api/data' && req.method === 'GET') {
res.setHeader('Content-Type', 'application/json');
const defaultApiInput = {
@ -14,21 +17,26 @@ const server = http.createServer(async (req, res) => {
},
},
};
// Get client ID and context from request headers
// Get client ID and context and nimbus preview flag from request headers
const clientID = req.headers['x-client-id'] || defaultApiInput.client_id;
const contextJSON = req.headers['x-context'] || JSON.stringify(defaultApiInput.context);
const context = JSON.parse(contextJSON);
const nimbusPreview = parsedUrl.query.nimbus_preview === 'true';
const apiInput = {
client_id: clientID,
context: context,
nimbus_preview: nimbusPreview,
};
let pathName = '/v1/features/'
if(nimbusPreview) {
pathName= pathName+"?nimbus_preview=" + nimbusPreview;
}
const options = {
hostname: 'cirrus', // Use the service name
hostname: 'cirrus',
port: 8001,
path: '/v1/features/',
path: pathName,
method: 'POST',
headers: {
'Content-Type': 'application/json',

83
experimenter/tests/integration/nimbus/pages/demo_app/frontend.py Normal file → Executable file
Просмотреть файл

@ -7,31 +7,74 @@ from nimbus.pages.base import Base
class DemoAppPage(Base):
PAGE_TITLE = "Demo app frontend"
CONTROL_BRANCH = "Control branch"
TREATMENT_BRANCH = "Treatment branch"
# XPaths for elements
CLIENT_ID_INPUT_XPATH = "//input[@placeholder='Client ID']"
CONTEXT_INPUT_XPATH = "//input[@placeholder='Context']"
PREVIEW_CHECKBOX_XPATH = "//input[@type='checkbox']"
SEND_DETAILS_BUTTON_XPATH = "//button[contains(text(), 'Send My Details')]"
RESULT_TEXT_XPATH_TEMPLATE = "//h1[{}]"
# Client dictionaries
CLIENT_1 = {"client_id": "dummy_client_id", "context": '{"test1":"test2"}'}
CLIENT_2 = {"client_id": "example1", "context": '{"test1":"test2"}'}
CLIENT_3 = {"client_id": "test", "context": '{"language":"en", "region":"CA"}'}
# Branch value constants
REFERENCE_BRANCH_VALUE = f'{{"enabled": true, "something": "{CONTROL_BRANCH}"}}'
TREATMENT_BRANCH_VALUE = f'{{"enabled": true, "something": "{TREATMENT_BRANCH}"}}'
# Default values
DEFAULT_NOT_ENROLLED_TEXT = "Not Enrolled"
DEFAULT_WICKED_TEXT = "wicked"
def wait_for_result_text(self, text_list):
xpath_conditions = " | ".join([f".='{text}'" for text in text_list])
xpath = f"//h1[{' or '.join(xpath_conditions.split(' | '))}]"
"""Wait for and return the result text element matching any text in text_list."""
xpath_conditions = " or ".join([f".='{text}'" for text in text_list])
xpath = self.RESULT_TEXT_XPATH_TEMPLATE.format(xpath_conditions)
return self.wait_for_and_find_element(By.XPATH, xpath, description="Result Text")
return self.wait_for_and_find_element(By.XPATH, xpath, description=None)
def fill_form(self, client_id, context):
"""Fill the form with client_id and context."""
self.enter_text(self.CLIENT_ID_INPUT_XPATH, client_id, "Client ID Input")
self.enter_text(self.CONTEXT_INPUT_XPATH, context, "Context Input")
def fill_and_send_form_data(self, client_id, context):
client_xpath = "//input[@placeholder='Client ID']"
client_id_input = self.wait_for_and_find_element(
By.XPATH, client_xpath, description=None
def enable_nimbus_preview(self):
"""Enable Nimbus Preview checkbox if not already selected."""
preview_checkbox = self.wait_for_and_find_element(
By.XPATH, self.PREVIEW_CHECKBOX_XPATH, description="Nimbus Preview Checkbox"
)
context_xpath = "//input[@placeholder='Context']"
context_input = self.wait_for_and_find_element(
By.XPATH, context_xpath, description=None
)
time.sleep(10)
client_id_input.send_keys(client_id)
context_input.send_keys(context)
if not preview_checkbox.is_selected():
preview_checkbox.click()
def click_send_my_details(self):
details_xpath = "//button[contains(text(), 'Send My Details')]"
send_details_button = self.wait_for_and_find_element(
By.XPATH, details_xpath, description=None
)
"""Click the 'Send My Details' button."""
self.click_button(self.SEND_DETAILS_BUTTON_XPATH, "Send My Details Button")
send_details_button.click()
def fill_and_send_form_data(self, client_id, context, nimbus_preview=False):
"""Fill the form with client_id and context, \
optionally enable Nimbus preview, and submit the form."""
self.fill_form(client_id, context)
if nimbus_preview:
self.enable_nimbus_preview()
time.sleep(10)
self.click_send_my_details()
def enter_text(self, xpath, text, description):
"""Helper method to enter text into an input field."""
input_element = self.wait_for_and_find_element(
By.XPATH, xpath, description=description
)
input_element.clear()
input_element.send_keys(text)
def click_button(self, xpath, description):
"""Helper method to click a button."""
button_element = self.wait_for_and_find_element(
By.XPATH, xpath, description=description
)
button_element.click()

213
experimenter/tests/integration/nimbus/test_cirrus_integration.py Normal file → Executable file
Просмотреть файл

@ -20,9 +20,10 @@ def test_create_new_rollout_approve_remote_settings_cirrus(
demo_app,
):
# Launch a rollout with 100% population
reference_branch_value = '{"enabled": true, "something": "You are enrolled"}'
create_experiment(
selenium, is_rollout=True, reference_branch_value=reference_branch_value
selenium,
is_rollout=True,
reference_branch_value=demo_app.REFERENCE_BRANCH_VALUE,
).launch_and_approve()
kinto_client.approve()
@ -31,20 +32,20 @@ def test_create_new_rollout_approve_remote_settings_cirrus(
HomePage(selenium, base_url).open().find_in_table(experiment_name)
# demo app frontend, default displays "Not Enrolled"
# Demo app frontend, default displays "Not Enrolled"
navigate_to(selenium)
selenium.refresh()
result_text_element = demo_app.wait_for_result_text(["Not Enrolled"])
assert result_text_element.is_displayed()
assert demo_app.wait_for_result_text(
[demo_app.DEFAULT_NOT_ENROLLED_TEXT]
).is_displayed()
# send client_id and context in a request to backend, backend will connect to
# cirrus and will return back the response
demo_app.fill_and_send_form_data("dummy_client_id", '{"test1":"test2"}')
demo_app.click_send_my_details()
# Send client_id and context to backend
demo_app.fill_and_send_form_data(
demo_app.CLIENT_1["client_id"], demo_app.CLIENT_1["context"]
)
# it should render "You are enrolled", reference branch value
result_text_element = demo_app.wait_for_result_text(["You are enrolled"])
assert result_text_element.is_displayed()
# Should render "You are enrolled"
assert demo_app.wait_for_result_text([demo_app.CONTROL_BRANCH]).is_displayed()
# Unenroll
summary = SummaryPage(selenium, experiment_url).open()
@ -56,17 +57,17 @@ def test_create_new_rollout_approve_remote_settings_cirrus(
# Demo app frontend, default displays "Not Enrolled" again
navigate_to(selenium)
selenium.refresh()
result_text_element = demo_app.wait_for_result_text(["Not Enrolled"])
assert result_text_element.is_displayed()
assert demo_app.wait_for_result_text(
[demo_app.DEFAULT_NOT_ENROLLED_TEXT]
).is_displayed()
# Send the same client_id and context after the rollout has ended
demo_app.fill_and_send_form_data("dummy_client_id", '{"test1":"test2"}')
demo_app.click_send_my_details()
demo_app.fill_and_send_form_data(
demo_app.CLIENT_1["client_id"], demo_app.CLIENT_1["context"]
)
# returns the default value
result_text_element = demo_app.wait_for_result_text(["wicked"])
assert result_text_element.is_displayed()
# Returns the default value
assert demo_app.wait_for_result_text([demo_app.DEFAULT_WICKED_TEXT]).is_displayed()
@pytest.mark.cirrus_enrollment
@ -80,12 +81,10 @@ def test_create_new_experiment_approve_remote_settings_cirrus(
demo_app,
):
# Launch an experiment with two branches
reference_branch_value = '{"enabled": true, "something": "Control branch"}'
treatment_branch_value = '{"enabled": true, "something": "Treatment branch"}'
create_experiment(
selenium,
reference_branch_value=reference_branch_value,
treatment_branch_value=treatment_branch_value,
reference_branch_value=demo_app.REFERENCE_BRANCH_VALUE,
treatment_branch_value=demo_app.TREATMENT_BRANCH_VALUE,
).launch_and_approve()
kinto_client.approve()
@ -98,33 +97,36 @@ def test_create_new_experiment_approve_remote_settings_cirrus(
navigate_to(selenium)
selenium.refresh()
result_text_element = demo_app.wait_for_result_text(["Not Enrolled"])
assert result_text_element.is_displayed()
assert demo_app.wait_for_result_text(
[demo_app.DEFAULT_NOT_ENROLLED_TEXT]
).is_displayed()
# Pass client_id and context
demo_app.fill_and_send_form_data("test", '{"test1":"test2"}')
demo_app.click_send_my_details()
demo_app.fill_and_send_form_data(
demo_app.CLIENT_1["client_id"], demo_app.CLIENT_1["context"]
)
# Determine the variation displayed and assert accordingly
# Check if either "Control branch" or "Treatment branch" is displayed
displayed_text = demo_app.wait_for_result_text(
["Control branch", "Treatment branch"]
[demo_app.CONTROL_BRANCH, demo_app.TREATMENT_BRANCH]
).text
assert displayed_text in ["Control branch", "Treatment branch"]
assert displayed_text in [demo_app.CONTROL_BRANCH, demo_app.TREATMENT_BRANCH]
# Refresh the page and try passing a new client
selenium.refresh()
result_text_element = demo_app.wait_for_result_text(["Not Enrolled"])
assert result_text_element.is_displayed()
assert demo_app.wait_for_result_text(
[demo_app.DEFAULT_NOT_ENROLLED_TEXT]
).is_displayed()
demo_app.fill_and_send_form_data("example1", '{"test1":"test2"}')
demo_app.click_send_my_details()
demo_app.fill_and_send_form_data(
demo_app.CLIENT_2["client_id"], demo_app.CLIENT_2["context"]
)
# Determine the variation displayed and assert accordingly
displayed_text = demo_app.wait_for_result_text(
["Control branch", "Treatment branch"]
[demo_app.CONTROL_BRANCH, demo_app.TREATMENT_BRANCH]
).text
assert displayed_text in ["Control branch", "Treatment branch"]
assert displayed_text in [demo_app.CONTROL_BRANCH, demo_app.TREATMENT_BRANCH]
# Unenroll
summary = SummaryPage(selenium, experiment_url).open()
@ -136,26 +138,26 @@ def test_create_new_experiment_approve_remote_settings_cirrus(
navigate_to(selenium)
selenium.refresh()
result_text_element = demo_app.wait_for_result_text(["Not Enrolled"])
assert result_text_element.is_displayed()
assert demo_app.wait_for_result_text(
[demo_app.DEFAULT_NOT_ENROLLED_TEXT]
).is_displayed()
# Send the same client_id and context after the experiment has ended
demo_app.fill_and_send_form_data("example1", '{"test1":"test2"}')
demo_app.click_send_my_details()
demo_app.fill_and_send_form_data(
demo_app.CLIENT_2["client_id"], demo_app.CLIENT_2["context"]
)
# returns the default value
result_text_element = demo_app.wait_for_result_text(["wicked"])
assert result_text_element.is_displayed()
# Returns the default value
assert demo_app.wait_for_result_text([demo_app.DEFAULT_WICKED_TEXT]).is_displayed()
# Check another client id
selenium.refresh()
# Send the same client_id and context after the experiment has ended
demo_app.fill_and_send_form_data("test", '{"test1":"test2"}')
demo_app.click_send_my_details()
demo_app.fill_and_send_form_data(
demo_app.CLIENT_1["client_id"], demo_app.CLIENT_1["context"]
)
# returns the default value
result_text_element = demo_app.wait_for_result_text(["wicked"])
assert result_text_element.is_displayed()
# Returns the default value
assert demo_app.wait_for_result_text([demo_app.DEFAULT_WICKED_TEXT]).is_displayed()
@pytest.mark.cirrus_enrollment
@ -169,12 +171,10 @@ def test_check_cirrus_targeting(
demo_app,
):
# Launch an experiment with two branches
reference_branch_value = '{"enabled": true, "something": "Control branch"}'
treatment_branch_value = '{"enabled": true, "something": "Treatment branch"}'
create_experiment(
selenium,
reference_branch_value=reference_branch_value,
treatment_branch_value=treatment_branch_value,
reference_branch_value=demo_app.REFERENCE_BRANCH_VALUE,
treatment_branch_value=demo_app.TREATMENT_BRANCH_VALUE,
languages=True,
countries=True,
).launch_and_approve()
@ -189,33 +189,36 @@ def test_check_cirrus_targeting(
navigate_to(selenium)
selenium.refresh()
result_text_element = demo_app.wait_for_result_text(["Not Enrolled"])
assert result_text_element.is_displayed()
assert demo_app.wait_for_result_text(
[demo_app.DEFAULT_NOT_ENROLLED_TEXT]
).is_displayed()
# Pass client_id and context
demo_app.fill_and_send_form_data("test", '{"language":"en", "region":"CA"}')
demo_app.click_send_my_details()
demo_app.fill_and_send_form_data(
demo_app.CLIENT_3["client_id"], demo_app.CLIENT_3["context"]
)
# Determine the variation displayed and assert accordingly
# Check if either "Control branch" or "Treatment branch" is displayed
displayed_text = demo_app.wait_for_result_text(
["Control branch", "Treatment branch"]
[demo_app.CONTROL_BRANCH, demo_app.TREATMENT_BRANCH]
).text
assert displayed_text in ["Control branch", "Treatment branch"]
assert displayed_text in [demo_app.CONTROL_BRANCH, demo_app.TREATMENT_BRANCH]
# Refresh the page and try passing a new client
selenium.refresh()
result_text_element = demo_app.wait_for_result_text(["Not Enrolled"])
assert result_text_element.is_displayed()
assert demo_app.wait_for_result_text(
[demo_app.DEFAULT_NOT_ENROLLED_TEXT]
).is_displayed()
demo_app.fill_and_send_form_data("example1", '{"language":"en", "region":"CA"}')
demo_app.click_send_my_details()
demo_app.fill_and_send_form_data(
demo_app.CLIENT_2["client_id"], demo_app.CLIENT_3["context"]
)
# Determine the variation displayed and assert accordingly
displayed_text = demo_app.wait_for_result_text(
["Control branch", "Treatment branch"]
[demo_app.CONTROL_BRANCH, demo_app.TREATMENT_BRANCH]
).text
assert displayed_text in ["Control branch", "Treatment branch"]
assert displayed_text in [demo_app.CONTROL_BRANCH, demo_app.TREATMENT_BRANCH]
# Unenroll
summary = SummaryPage(selenium, experiment_url).open()
@ -227,23 +230,73 @@ def test_check_cirrus_targeting(
navigate_to(selenium)
selenium.refresh()
result_text_element = demo_app.wait_for_result_text(["Not Enrolled"])
assert result_text_element.is_displayed()
assert demo_app.wait_for_result_text(
[demo_app.DEFAULT_NOT_ENROLLED_TEXT]
).is_displayed()
# Send the same client_id and context after the experiment has ended
demo_app.fill_and_send_form_data("example1", '{"language":"en", "region":"CA"}')
demo_app.click_send_my_details()
demo_app.fill_and_send_form_data(
demo_app.CLIENT_3["client_id"], demo_app.CLIENT_3["context"]
)
# returns the default value
result_text_element = demo_app.wait_for_result_text(["wicked"])
assert result_text_element.is_displayed()
# Returns the default value
assert demo_app.wait_for_result_text([demo_app.DEFAULT_WICKED_TEXT]).is_displayed()
# Check another client id
selenium.refresh()
# Send the same client_id and context after the experiment has ended
demo_app.fill_and_send_form_data("test", '{"language":"en", "region":"CA"}')
demo_app.click_send_my_details()
demo_app.fill_and_send_form_data(
demo_app.CLIENT_1["client_id"], demo_app.CLIENT_3["context"]
)
# returns the default value
result_text_element = demo_app.wait_for_result_text(["wicked"])
assert result_text_element.is_displayed()
# Returns the default value
assert demo_app.wait_for_result_text([demo_app.DEFAULT_WICKED_TEXT]).is_displayed()
@pytest.mark.cirrus_enrollment
def test_nimbus_preview_flag(
selenium,
experiment_url,
create_experiment,
kinto_client,
base_url,
experiment_name,
demo_app,
):
create_experiment(
selenium,
reference_branch_value=demo_app.REFERENCE_BRANCH_VALUE,
treatment_branch_value=demo_app.TREATMENT_BRANCH_VALUE,
).launch_to_preview()
SummaryPage(selenium, experiment_url).open().wait_for_preview_status()
navigate_to(selenium)
selenium.refresh()
assert demo_app.wait_for_result_text(
[demo_app.DEFAULT_NOT_ENROLLED_TEXT]
).is_displayed()
# Enable nimbus preview flag
demo_app.fill_and_send_form_data(
demo_app.CLIENT_1["client_id"],
demo_app.CLIENT_1["context"],
nimbus_preview=True,
)
displayed_text = demo_app.wait_for_result_text(
[demo_app.CONTROL_BRANCH, demo_app.TREATMENT_BRANCH]
).text
assert displayed_text in [demo_app.CONTROL_BRANCH, demo_app.TREATMENT_BRANCH]
navigate_to(selenium)
selenium.refresh()
assert demo_app.wait_for_result_text(
[demo_app.DEFAULT_NOT_ENROLLED_TEXT]
).is_displayed()
# Not using nimbus preview flag
demo_app.fill_and_send_form_data(
demo_app.CLIENT_2["client_id"], demo_app.CLIENT_2["context"]
)
assert demo_app.wait_for_result_text([demo_app.DEFAULT_WICKED_TEXT]).is_displayed()