fix #9239 feat(schemas): add factories to generate test data (#9240)

Because

- we don't want to have to update static test data whenever schemas
change
- we do still want tests on schemas package consumers to test overall
functionality

This commit

- adds some test data generation factories to the schemas package for
consumers to use within tests

schemas/mozilla_nimbus_schemas/jetstream/__init__.py

@@ -1,7 +1,8 @@
-from .analysis_errors import AnalysisError, AnalysisErrors
-from .metadata import ExternalConfig, Metadata, Metric, Outcome
+from .analysis_errors import AnalysisError, AnalysisErrors, AnalysisErrorsFactory
+from .metadata import ExternalConfig, Metadata, MetadataFactory, Metric, Outcome
 from .population_sizing import (
     SampleSizes,
+    SampleSizesFactory,
     SizingByUserType,
     SizingDetails,
     SizingMetric,
@@ -12,7 +13,7 @@ from .population_sizing import (
     SizingTarget,
     SizingUserType,
 )
-from .statistics import AnalysisBasis, Statistic, Statistics
+from .statistics import AnalysisBasis, Statistic, Statistics, StatisticsFactory
 
 __all__ = [
     "AnalysisBasis",
@@ -34,4 +35,8 @@ __all__ = [
     "SizingUserType",
     "Statistic",
     "Statistics",
+    "AnalysisErrorsFactory",
+    "MetadataFactory",
+    "SampleSizesFactory",
+    "StatisticsFactory",
 ]

schemas/mozilla_nimbus_schemas/jetstream/analysis_errors.py

@@ -1,6 +1,7 @@
 import datetime as dt
 from typing import Optional
 
+from polyfactory.factories.pydantic_factory import ModelFactory
 from pydantic import BaseModel
 
 from mozilla_nimbus_schemas.jetstream.statistics import AnalysisBasis
@@ -23,3 +24,7 @@ class AnalysisError(BaseModel):
 
 class AnalysisErrors(BaseModel):
     __root__: list[AnalysisError]
+
+
+class AnalysisErrorsFactory(ModelFactory[AnalysisErrors]):
+    __model__ = AnalysisErrors

schemas/mozilla_nimbus_schemas/jetstream/metadata.py

@@ -2,6 +2,7 @@ import datetime as dt
 import json
 from typing import Optional
 
+from polyfactory.factories.pydantic_factory import ModelFactory
 from pydantic import BaseModel, HttpUrl, validator
 
 from mozilla_nimbus_schemas.jetstream.statistics import SCHEMA_VERSION, AnalysisBasis
@@ -55,3 +56,7 @@ class Metadata(BaseModel):
     class Config:
         # override json_loads because `description` field in Metric may contain \n
         json_loads = nonstrict_json_loads
+
+
+class MetadataFactory(ModelFactory[Metadata]):
+    __model__ = Metadata

schemas/mozilla_nimbus_schemas/jetstream/population_sizing.py

@@ -1,5 +1,6 @@
 from enum import Enum
 
+from polyfactory.factories.pydantic_factory import ModelFactory
 from pydantic import BaseModel
 
 
@@ -57,3 +58,7 @@ class SizingByUserType(BaseModel):
 class SampleSizes(BaseModel):
     # dynamic key representing the target for easy lookup
     __root__: dict[str, SizingByUserType]
+
+
+class SampleSizesFactory(ModelFactory[SampleSizes]):
+    __model__ = SampleSizes

schemas/mozilla_nimbus_schemas/jetstream/statistics.py

@@ -1,6 +1,7 @@
 from enum import Enum
 from typing import Optional
 
+from polyfactory.factories.pydantic_factory import ModelFactory
 from pydantic import BaseModel, Field
 
 SCHEMA_VERSION = 4
@@ -29,3 +30,7 @@ class Statistic(BaseModel):
 
 class Statistics(BaseModel):
     __root__: list[Statistic]
+
+
+class StatisticsFactory(ModelFactory[Statistics]):
+    __model__ = Statistics

schemas/mozilla_nimbus_schemas/tests/jetstream/test_analysis_errors.py

@@ -5,6 +5,7 @@ from mozilla_nimbus_schemas.jetstream import (
     AnalysisBasis,
     AnalysisError,
     AnalysisErrors,
+    AnalysisErrorsFactory,
 )
 
 """
@@ -128,3 +129,16 @@ def test_parse_analysis_errors_fails():
     """
     with pytest.raises(ValidationError):
         AnalysisErrors.parse_raw(error_json)
+
+
+def test_analysis_errors_factory():
+    analysis_errors = AnalysisErrorsFactory.build()
+    AnalysisErrors.validate(analysis_errors)
+
+
+def test_analysis_errors_invalid():
+    analysis_errors_dict = AnalysisErrorsFactory.build().dict()
+    print(analysis_errors_dict)
+    analysis_errors_dict["__root__"][0]["timestamp"] = "not a date!"
+    with pytest.raises(ValidationError):
+        AnalysisErrors.parse_obj(analysis_errors_dict)

schemas/mozilla_nimbus_schemas/tests/jetstream/test_metadata.py

@@ -1,6 +1,9 @@
 import datetime as dt
 
-from mozilla_nimbus_schemas.jetstream import Metadata
+import pytest
+from pydantic import ValidationError
+
+from mozilla_nimbus_schemas.jetstream import Metadata, MetadataFactory
 
 """
 Test cases for metadata schemas:
@@ -63,3 +66,16 @@ def test_parse_metadata():
     assert metadata.analysis_start_time == dt.datetime(
         2023, 5, 1, 1, 2, 3, tzinfo=dt.timezone.utc
     )
+
+
+def test_metadata_factory():
+    metadata = MetadataFactory.build()
+    Metadata.validate(metadata)
+
+
+def test_metadata_invalid():
+    metadata_dict = MetadataFactory.build().dict()
+    print(metadata_dict)
+    metadata_dict["schema_version"] = "not a number"
+    with pytest.raises(ValidationError):
+        Metadata.parse_obj(metadata_dict)

schemas/mozilla_nimbus_schemas/tests/jetstream/test_population_sizing.py

@@ -1,7 +1,13 @@
 import json
 from unittest import TestCase
 
-from mozilla_nimbus_schemas.jetstream import SampleSizes
+import pytest
+from pydantic import ValidationError
+
+from mozilla_nimbus_schemas.jetstream import (
+    SampleSizes,
+    SampleSizesFactory,
+)
 
 """
 Test cases for population sizing schemas.
@@ -87,3 +93,14 @@ class TestPopulationSizing(TestCase):
         """
         sample_sizes = SampleSizes.parse_raw(sizing_test_data)
         self.assertEqual(json.dumps(json.loads(sizing_test_data)), sample_sizes.json())
+
+    def test_parse_population_sizing_factory(self):
+        sample_sizes = SampleSizesFactory.build()
+        SampleSizes.validate(sample_sizes)
+
+    def test_parse_population_sizing_factory_invalid(self):
+        sample_sizes_dict = SampleSizesFactory.build().dict()
+        first_key = list(sample_sizes_dict.keys())[0]
+        sample_sizes_dict[first_key]["BAD"] = {}
+        with pytest.raises(ValidationError):
+            SampleSizes.parse_obj(sample_sizes_dict)

schemas/mozilla_nimbus_schemas/tests/jetstream/test_statistics.py

@@ -1,4 +1,12 @@
-from mozilla_nimbus_schemas.jetstream import AnalysisBasis, Statistic, Statistics
+import pytest
+from pydantic import ValidationError
+
+from mozilla_nimbus_schemas.jetstream import (
+    AnalysisBasis,
+    Statistic,
+    Statistics,
+    StatisticsFactory,
+)
 
 """
 Test cases for statistics schemas:
@@ -90,3 +98,15 @@ def test_parse_statistics():
     """
     stats = Statistics.parse_raw(stats_json)
     assert len(stats.__root__) == 3
+
+
+def test_statistics_factory():
+    stats = StatisticsFactory.build()
+    Statistics.validate(stats)
+
+
+def test_statistics_invalid():
+    stats_dict = StatisticsFactory.build().dict()
+    stats_dict["__root__"][0]["ci_width"] = "invalid data"
+    with pytest.raises(ValidationError):
+        Statistics.parse_obj(stats_dict)

schemas/poetry.lock

@@ -329,6 +329,20 @@ files = [
 [package.extras]
 test = ["pytest (>=6)"]
 
+[[package]]
+name = "faker"
+version = "19.3.0"
+description = "Faker is a Python package that generates fake data for you."
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "Faker-19.3.0-py3-none-any.whl", hash = "sha256:bee54278d6e1289573317604ab6f4782acca724396bf261eaf1890de228e553d"},
+    {file = "Faker-19.3.0.tar.gz", hash = "sha256:7d6ed00de3eef9bd57504500c67ee034cab959e4248f9c24aca33e08af82ca93"},
+]
+
+[package.dependencies]
+python-dateutil = ">=2.4"
+
 [[package]]
 name = "idna"
 version = "3.4"
@@ -549,6 +563,29 @@ files = [
 dev = ["pre-commit", "tox"]
 testing = ["pytest", "pytest-benchmark"]
 
+[[package]]
+name = "polyfactory"
+version = "2.7.2"
+description = "Mock data generation factories"
+optional = false
+python-versions = ">=3.8,<4.0"
+files = [
+    {file = "polyfactory-2.7.2-py3-none-any.whl", hash = "sha256:4e61d504527368a90bc51423670ba3528304d20ae0fb9705464ee99a40a711ca"},
+    {file = "polyfactory-2.7.2.tar.gz", hash = "sha256:ec919ccca990987134ff8f7b400983feb54285f832b743aa7012be35f998ccd4"},
+]
+
+[package.dependencies]
+faker = "*"
+typing-extensions = "*"
+
+[package.extras]
+attrs = ["attrs"]
+beanie = ["beanie", "pydantic[email]"]
+full = ["attrs", "beanie", "msgspec", "odmantic", "pydantic[email]"]
+msgspec = ["msgspec"]
+odmantic = ["odmantic", "pydantic[email]"]
+pydantic = ["pydantic[email]"]
+
 [[package]]
 name = "pycparser"
 version = "2.21"
@@ -648,6 +685,20 @@ tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""}
 [package.extras]
 testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"]
 
+[[package]]
+name = "python-dateutil"
+version = "2.8.2"
+description = "Extensions to the standard Python datetime module"
+optional = false
+python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
+files = [
+    {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"},
+    {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"},
+]
+
+[package.dependencies]
+six = ">=1.5"
+
 [[package]]
 name = "pywin32-ctypes"
 version = "0.2.0"
@@ -887,4 +938,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.10"
-content-hash = "a7bb3892f708aba11fb15b2674bf4284b22e8f0b34b6e7aaab1c3cf636c54bb0"
+content-hash = "d9b8c79f17d9d5f4f5401d1448aebbee01979274faacd3384927f46cbd2cc848"

schemas/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "mozilla-nimbus-schemas"
-version = "2023.8.1"
+version = "2023.8.2"
 description = "Schemas used by Mozilla Nimbus and related projects."
 authors = ["mikewilli"]
 license = "MPL 2.0"
@@ -10,6 +10,7 @@ packages = [{include = "mozilla_nimbus_schemas"}]
 [tool.poetry.dependencies]
 python = "^3.10"
 pydantic = "^1.10.7"
+polyfactory = "^2.7.2"
 
 [tool.poetry.group.dev.dependencies]
 ruff = "^0.0.269"