diff --git a/automation/python-tests/conftest.py b/automation/python-tests/conftest.py index 6060b0c88..419c29d63 100644 --- a/automation/python-tests/conftest.py +++ b/automation/python-tests/conftest.py @@ -94,24 +94,8 @@ def client(app_context, experiment): return c -@pytest.fixture(scope="class") -def cirrus_client(request): - app_context = json.dumps( - { - "app_id": "test app id", - "app_name": "test app name", - "channel": "dev", - } - ) - - bucket_config = { - "randomizationUnit": "user_id", - "count": 100, - "namespace": "", - "start": 1, - "total": 100, - } - +@pytest.fixture +def cirrus_client(app_context, bucket_config): branches = [ { "slug": "control", @@ -149,14 +133,15 @@ def cirrus_client(request): "featureIds": ["imported-module-1-included-feature-1"], } - request.cls.cirrus_client = CirrusClient(app_context) + client = CirrusClient(app_context) data = json.dumps({"data": [experiment]}) - request.cls.cirrus_client.set_experiments(data) + client.set_experiments(data) + return client -@pytest.fixture(scope="class") -def fml_client(request): - def _client(_, path, channel): +@pytest.fixture +def fml_client(): + def _client(path, channel): return FmlClient("./automation/python-tests/resources/" + path, channel) - request.cls.fml_client = _client + return _client diff --git a/automation/python-tests/test_cirrus_fml_integration.py b/automation/python-tests/test_cirrus_fml_integration.py index 1377f4708..ce1379dc6 100644 --- a/automation/python-tests/test_cirrus_fml_integration.py +++ b/automation/python-tests/test_cirrus_fml_integration.py @@ -1,85 +1,74 @@ import json -import pytest -import unittest -@pytest.mark.usefixtures("fml_client", "cirrus_client") -class TestCirrusClientFmlClientIntegration(unittest.TestCase): - def test_enroll_and_get_enrolled_feature_json(self): - # create request - req_1 = json.dumps( - { - "clientId": "jeddai", - "requestContext": {"username": "jeddai"}, - } - ) - # enroll, convert response to dict from JSON - res_1 = json.loads(self.cirrus_client.handle_enrollment(req_1)) - # map enrolledFeatureConfigMap values to the feature values - feature_configs = json.dumps( - [value["feature"] for value in res_1["enrolledFeatureConfigMap"].values()] - ) +# 1. create request +# 2. enroll, convert response to dict from JSON +# 3. map enrolledFeatureConfigMap values to the feature values +# 4. create FmlClient +# 5. merge feature configs into default JSON and validate their values against the manifest +# 6. load JSON response as dict +def test_enroll_and_get_enrolled_feature_json_control(fml_client, cirrus_client): + req = json.dumps( + { + "clientId": "jeddai", + "requestContext": {"username": "jeddai"}, + } + ) + res = json.loads(cirrus_client.handle_enrollment(req)) + feature_configs = json.dumps( + [value["feature"] for value in res["enrolledFeatureConfigMap"].values()] + ) - assert ( - res_1["enrolledFeatureConfigMap"]["imported-module-1-included-feature-1"][ - "slug" - ] - == "experiment-slug" - ) - assert ( - res_1["enrolledFeatureConfigMap"]["imported-module-1-included-feature-1"][ - "branch" - ] - == "control" - ) + assert ( + res["enrolledFeatureConfigMap"]["imported-module-1-included-feature-1"]["slug"] + == "experiment-slug" + ) + assert ( + res["enrolledFeatureConfigMap"]["imported-module-1-included-feature-1"][ + "branch" + ] + == "control" + ) - # create FmlClient - fml_client = self.fml_client("test-include-import.fml.yml", "developer") - # merge feature configs into default JSON and validate their values against the manifest - merged_res_1 = fml_client.validate_feature_configs_and_merge_into_defaults( - feature_configs - ) - # load JSON response as dict - merged_res_json_1 = json.loads(merged_res_1.json) + fml_client = fml_client("test-include-import.fml.yml", "developer") + merged_res = fml_client.validate_feature_configs_and_merge_into_defaults( + feature_configs + ) + merged_res_json = json.loads(merged_res.json) - assert ( - merged_res_json_1["imported-module-1-included-feature-1"]["enabled"] - is False - ) - assert len(merged_res_1.errors) == 0 + assert merged_res_json["imported-module-1-included-feature-1"]["enabled"] is False + assert len(merged_res.errors) == 0 - # repeat the above but with a different client/username on the request - req_2 = json.dumps( - { - "clientId": "test", - "requestContext": {"username": "test"}, - } - ) - res_2 = json.loads(self.cirrus_client.handle_enrollment(req_2)) - feature_configs = json.dumps( - [value["feature"] for value in res_2["enrolledFeatureConfigMap"].values()] - ) - assert ( - res_2["enrolledFeatureConfigMap"]["imported-module-1-included-feature-1"][ - "slug" - ] - == "experiment-slug" - ) - assert ( - res_2["enrolledFeatureConfigMap"]["imported-module-1-included-feature-1"][ - "branch" - ] - == "treatment" - ) +# repeat the above but with a different client/username on the request +def test_enroll_and_get_enrolled_feature_json_treatment(fml_client, cirrus_client): + req = json.dumps( + { + "clientId": "test", + "requestContext": {"username": "test"}, + } + ) + res = json.loads(cirrus_client.handle_enrollment(req)) + feature_configs = json.dumps( + [value["feature"] for value in res["enrolledFeatureConfigMap"].values()] + ) - fml_client = self.fml_client("test-include-import.fml.yml", "developer") - merged_res_2 = fml_client.validate_feature_configs_and_merge_into_defaults( - feature_configs - ) - merged_res_json_2 = json.loads(merged_res_2.json) + assert ( + res["enrolledFeatureConfigMap"]["imported-module-1-included-feature-1"]["slug"] + == "experiment-slug" + ) + assert ( + res["enrolledFeatureConfigMap"]["imported-module-1-included-feature-1"][ + "branch" + ] + == "treatment" + ) - assert ( - merged_res_json_2["imported-module-1-included-feature-1"]["enabled"] is True - ) - assert len(merged_res_2.errors) == 0 + fml_client = fml_client("test-include-import.fml.yml", "developer") + merged_res = fml_client.validate_feature_configs_and_merge_into_defaults( + feature_configs + ) + merged_res_json = json.loads(merged_res.json) + + assert merged_res_json["imported-module-1-included-feature-1"]["enabled"] is True + assert len(merged_res.errors) == 0 diff --git a/automation/python-tests/test_fml.py b/automation/python-tests/test_fml.py index a407f53d2..6f64282fa 100644 --- a/automation/python-tests/test_fml.py +++ b/automation/python-tests/test_fml.py @@ -1,107 +1,116 @@ import json import pytest -import unittest from fml import FmlError, InternalError -@pytest.mark.usefixtures("fml_client") -class TestFmlClient(unittest.TestCase): - def test_instantiate_fml_client(self): - self.fml_client("test.fml.yml", "developer") +def test_instantiate_fml_client(fml_client): + fml_client("test.fml.yml", "developer") - def test_instantiate_fml_client_fails_if_invalid_path(self): - with self.assertRaises(FmlError): - self.fml_client("a-random-path", "developer") - def test_instantiate_fml_client_fails_if_invalid_yml(self): - with self.assertRaises(InternalError): - self.fml_client("test-invalid.fml.yml", "developer") +def test_instantiate_fml_client_fails_if_invalid_path(fml_client): + with pytest.raises(FmlError): + fml_client("a-random-path", "developer") - def test_instantiate_fml_client_fails_if_invalid_channel(self): - with self.assertRaises(FmlError): - self.fml_client("test.fml.yml", "release") - def test_default_json(self): - client = self.fml_client("test.fml.yml", "developer") - defaults = json.loads(client.get_default_json()) - assert defaults["example-feature"]["enabled"] is False - assert defaults["example-feature"]["something"] == "wicked" +def test_instantiate_fml_client_fails_if_invalid_yml(fml_client): + with pytest.raises(InternalError): + fml_client("test-invalid.fml.yml", "developer") - client = self.fml_client("test.fml.yml", "nightly") - defaults = json.loads(client.get_default_json()) - assert defaults["example-feature"]["enabled"] is True - assert defaults["example-feature"].get("something") is None - def test_validate(self): - client = self.fml_client("test.fml.yml", "developer") - config = {"featureId": "example-feature", "value": {"enabled": True}} - assert client.validate_feature_config(json.dumps(config)) is True +def test_instantiate_fml_client_fails_if_invalid_channel(fml_client): + with pytest.raises(FmlError): + fml_client("test.fml.yml", "release") - def test_validate_false(self): - client = self.fml_client("test.fml.yml", "developer") - config = {"featureId": "example-featurea", "value": {"enabled": True}} - with self.assertRaises(FmlError): - client.validate_feature_config(json.dumps(config)) +def test_default_json(fml_client): + client = fml_client("test.fml.yml", "developer") + defaults = json.loads(client.get_default_json()) + assert defaults["example-feature"]["enabled"] is False + assert defaults["example-feature"]["something"] == "wicked" - def test_merge_and_validate(self): - client = self.fml_client("test.fml.yml", "developer") - configs = [{"featureId": "example-feature", "value": {"enabled": True}}] - result = client.validate_feature_configs_and_merge_into_defaults( - json.dumps(configs) - ) - assert len(result.errors) == 0 + client = fml_client("test.fml.yml", "nightly") + defaults = json.loads(client.get_default_json()) + assert defaults["example-feature"]["enabled"] is True + assert defaults["example-feature"].get("something") is None - result_json = json.loads(result.json)["example-feature"] - assert result_json["enabled"] is True - assert result_json["something"] == "wicked" - @pytest.mark.skip(reason="This functionality is hindered by EXP-3503") - def test_merge_and_validate_error_on_invalid_key(self): - client = self.fml_client("test.fml.yml", "developer") - configs = [{"featureId": "example-feature", "value": {"enabled1": False}}] - result = client.validate_feature_configs_and_merge_into_defaults( - json.dumps(configs) - ) +def test_validate_single_feature(fml_client): + client = fml_client("test.fml.yml", "developer") + config = {"featureId": "example-feature", "value": {"enabled": True}} + assert client.validate_feature_config(json.dumps(config)) is True - assert len(result.errors) == 1 - assert isinstance(result.errors[0], FmlError) - def test_merge_and_validate_error_on_invalid_value(self): - client = self.fml_client("test.fml.yml", "developer") - configs = [{"featureId": "example-feature", "value": {"enabled": "false"}}] - result = client.validate_feature_configs_and_merge_into_defaults( - json.dumps(configs) - ) +def test_validate_single_feature_false_invalid_feature(fml_client): + client = fml_client("test.fml.yml", "developer") + config = {"featureId": "example-featurea", "value": {"enabled": True}} - assert len(result.errors) == 1 - assert isinstance(result.errors[0], FmlError) + with pytest.raises(FmlError): + client.validate_feature_config(json.dumps(config)) - def test_merge_and_validate_on_included_and_imported_features(self): - client = self.fml_client( - "test-include-import.fml.yml", - "developer", - ) - configs = [ - {"featureId": "example-feature", "value": {"enabled": True}}, - {"featureId": "included-feature-1", "value": {"enabled": True}}, - ] - result = client.validate_feature_configs_and_merge_into_defaults( - json.dumps(configs) - ) - assert len(result.errors) == 0 +def test_merge_and_validate(fml_client): + client = fml_client("test.fml.yml", "developer") + configs = [{"featureId": "example-feature", "value": {"enabled": True}}] + result = client.validate_feature_configs_and_merge_into_defaults( + json.dumps(configs) + ) + assert len(result.errors) == 0 - example_feature = json.loads(result.json)["example-feature"] - assert example_feature["enabled"] is True - assert example_feature["something"] == "wicked" - included_feature_1 = json.loads(result.json)["included-feature-1"] - assert included_feature_1["enabled"] is True - imported_module_1_feature_1 = json.loads(result.json)[ - "imported-module-1-feature-1" - ] - assert imported_module_1_feature_1["enabled"] is True - imported_module_1_included_feature_1 = json.loads(result.json)[ - "imported-module-1-included-feature-1" - ] - assert imported_module_1_included_feature_1["enabled"] is False + result_json = json.loads(result.json)["example-feature"] + assert result_json["enabled"] is True + assert result_json["something"] == "wicked" + + +@pytest.mark.skip(reason="This functionality is hindered by EXP-3503") +def test_merge_and_validate_error_on_invalid_key(fml_client): + client = fml_client("test.fml.yml", "developer") + configs = [{"featureId": "example-feature", "value": {"enabled1": False}}] + result = client.validate_feature_configs_and_merge_into_defaults( + json.dumps(configs) + ) + + assert len(result.errors) == 1 + assert isinstance(result.errors[0], FmlError) + + +def test_merge_and_validate_error_on_invalid_value(fml_client): + client = fml_client("test.fml.yml", "developer") + configs = [{"featureId": "example-feature", "value": {"enabled": "false"}}] + result = client.validate_feature_configs_and_merge_into_defaults( + json.dumps(configs) + ) + + assert len(result.errors) == 1 + assert isinstance(result.errors[0], FmlError) + + +def test_merge_and_validate_on_included_and_imported_features(fml_client): + client = fml_client( + "test-include-import.fml.yml", + "developer", + ) + configs = [ + {"featureId": "example-feature", "value": {"enabled": True}}, + {"featureId": "included-feature-1", "value": {"enabled": True}}, + { + "featureId": "imported-module-1-included-feature-1", + "value": {"enabled": True}, + }, + ] + result = client.validate_feature_configs_and_merge_into_defaults( + json.dumps(configs) + ) + + assert len(result.errors) == 0 + + example_feature = json.loads(result.json)["example-feature"] + assert example_feature["enabled"] is True + assert example_feature["something"] == "wicked" + included_feature_1 = json.loads(result.json)["included-feature-1"] + assert included_feature_1["enabled"] is True + imported_module_1_feature_1 = json.loads(result.json)["imported-module-1-feature-1"] + assert imported_module_1_feature_1["enabled"] is True + imported_module_1_included_feature_1 = json.loads(result.json)[ + "imported-module-1-included-feature-1" + ] + assert imported_module_1_included_feature_1["enabled"] is True