diff --git a/schemas/VERSION b/schemas/VERSION index 2b9900ecf..795245549 100644 --- a/schemas/VERSION +++ b/schemas/VERSION @@ -1 +1 @@ -2024.11.3 +2024.11.4 diff --git a/schemas/index.d.ts b/schemas/index.d.ts index 2daa73766..d81efa657 100644 --- a/schemas/index.d.ts +++ b/schemas/index.d.ts @@ -6,13 +6,13 @@ * make schemas_build */ -export type DesktopApplication = "firefox-desktop" | "firefox-desktop-background-task"; -export type FeatureVariableType = "int" | "string" | "boolean" | "json"; -export type PrefBranch = "default" | "user"; /** * A unique, stable indentifier for the user used as an input to bucket hashing. */ export type RandomizationUnit = "normandy_id" | "nimbus_id" | "user_id" | "group_id"; +export type DesktopApplication = "firefox-desktop" | "firefox-desktop-background-task"; +export type FeatureVariableType = "int" | "string" | "boolean" | "json"; +export type PrefBranch = "default" | "user"; export type AnalysisBasis = "enrollments" | "exposures"; export type LogSource = "jetstream" | "sizing" | "jetstream-preview"; export type AnalysisErrors = AnalysisError[]; @@ -32,6 +32,237 @@ export type SizingMetricName = "active_hours" | "search_count" | "days_of_use" | export type StatisticIngestEnum = "percentage" | "binomial" | "mean" | "count"; export type Statistics = Statistic[]; +/** + * A Nimbus experiment for Firefox Desktop. + * + * This schema is more strict than DesktopNimbusExperiment and is backwards + * comaptible with Firefox Desktop versions less than 95. It is intended for use inside + * Experimenter itself. + */ +export interface DesktopAllVersionsNimbusExperiment { + /** + * Version of the NimbusExperiment schema this experiment refers to + */ + schemaVersion: string; + /** + * Unique identifier for the experiment + */ + slug: string; + /** + * Unique identifier for the experiiment. + * + * This is a duplicate of slug, but is required field for all Remote Settings records. + */ + id: string; + /** + * A slug identifying the targeted product of this experiment. + * + * It should be a lowercased_with_underscores name that is short and unambiguous and it should match the app_name found in https://probeinfo.telemetry.mozilla.org/glean/repositories. Examples are "fenix" and "firefox_desktop". + */ + appName: string; + /** + * The platform identifier for the targeted app. + * + * This should match app's identifier exactly as it appears in the relevant app store listing (for relevant platforms) or the app's Glean initialization (for other platforms). + * + * Examples are "org.mozilla.firefox_beta" and "firefox-desktop". + */ + appId: string; + /** + * A specific channel of an application such as "nightly", "beta", or "release". + */ + channel: string; + /** + * Public name of the experiment that will be displayed on "about:studies". + */ + userFacingName: string; + /** + * Short public description of the experiment that will be displayed on "about:studies". + */ + userFacingDescription: string; + /** + * When this property is set to true, the SDK should not enroll new users into the experiment that have not already been enrolled. + */ + isEnrollmentPaused: boolean; + /** + * When this property is set to true, treat this experiment as a rollout. + * + * Rollouts are currently handled as single-branch experiments separated from the bucketing namespace for normal experiments. + * + * See-also: https://mozilla-hub.atlassian.net/browse/SDK-405 + */ + isRollout?: boolean; + bucketConfig: ExperimentBucketConfig; + /** + * A list of outcomes relevant to the experiment analysis. + */ + outcomes?: ExperimentOutcome[]; + /** + * A list of featureIds the experiment contains configurations for. + */ + featureIds?: string[]; + /** + * A JEXL targeting expression used to filter out experiments. + */ + targeting?: string | null; + /** + * Actual publish date of the experiment. + * + * Note that this value is expected to be null in Remote Settings. + */ + startDate: string | null; + /** + * Actual enrollment end date of the experiment. + * + * Note that this value is expected to be null in Remote Settings. + */ + enrollmentEndDate?: string | null; + /** + * Actual end date of this experiment. + * + * Note that this field is expected to be null in Remote Settings. + */ + endDate: string | null; + /** + * Duration of the experiment from the start date in days. + * + * Note that this property is only used during the analysis phase (i.e., not by the SDK). + */ + proposedDuration?: number; + /** + * This represents the number of days that we expect to enroll new users. + * + * Note that this property is only used during the analysis phase (i.e., not by the SDK). + */ + proposedEnrollment: number; + /** + * The slug of the reference branch (i.e., the branch we consider "control"). + */ + referenceBranch: string | null; + /** + * The list of locale codes (e.g., "en-US" or "fr") that this experiment is targeting. + * + * If null, all locales are targeted. + */ + locales?: string[] | null; + /** + * The date that this experiment was first published to Remote Settings. + * + * If null, it has not yet been published. + */ + publishedDate?: string | null; + /** + * Branch configuration for the experiment. + */ + branches: DesktopAllVersionsExperimentBranch[]; + /** + * When this property is set to true, treat this experiment as aFirefox Labs experiment + */ + isFirefoxLabsOptIn?: boolean; + /** + * An optional string containing the Fluent ID for the title of the opt-in + */ + firefoxLabsTitle?: string; + /** + * An optional string containing the Fluent ID for the description of the opt-in + */ + firefoxLabsDescription?: string; + /** + * Opt out of feature schema validation. + */ + featureValidationOptOut?: boolean; + localizations?: ExperimentLocalizations | null; +} +export interface ExperimentBucketConfig { + randomizationUnit: RandomizationUnit; + /** + * Additional inputs to the hashing function. + */ + namespace: string; + /** + * Index of the starting bucket of the range. + */ + start: number; + /** + * Number of buckets in the range. + */ + count: number; + /** + * The total number of buckets. + * + * You can assume this will always be 10000 + */ + total: number; +} +export interface ExperimentOutcome { + /** + * Identifier for the outcome. + */ + slug: string; + /** + * e.g., "primary" or "secondary". + */ + priority: string; +} +/** + * The branch definition supported on all Firefox Desktop versions. + * + * This version requires the feature field to be present to support older Firefox Desktop + * clients. + */ +export interface DesktopAllVersionsExperimentBranch { + /** + * Identifier for the branch. + */ + slug: string; + /** + * Relative ratio of population for the branch. + * + * e.g., if branch A=1 and branch B=3, then branch A would get 25% of the population. + */ + ratio: number; + /** + * An array of feature configurations. + */ + features: ExperimentFeatureConfig[]; + /** + * An optional string containing the title of the branch + */ + firefoxLabsTitle?: string; + feature: DesktopPre95FeatureConfig; +} +export interface ExperimentFeatureConfig { + /** + * The identifier for the feature flag. + */ + featureId: string; + /** + * The values that define the feature configuration. + * + * This should be validated against a schema. + */ + value: { + [k: string]: unknown; + }; +} +export interface DesktopPre95FeatureConfig { + featureId: "this-is-included-for-desktop-pre-95-support"; + value: { + [k: string]: unknown; + }; + enabled: false; +} +/** + * Per-locale localization substitutions. + * + * The top level key is the locale (e.g., "en-US" or "fr"). Each entry is a mapping of + * string IDs to their localized equivalents. + */ +export interface ExperimentLocalizations { + [k: string]: { + [k: string]: string; + }; +} /** * A feature. */ @@ -135,6 +366,9 @@ export interface DesktopFeatureManifest { } /** * A Nimbus experiment for Firefox Desktop. + * + * This schema is less strict than DesktopAllVersionsNimbusExperiment and is intended for + * use in Firefox Desktop. */ export interface DesktopNimbusExperiment { /** @@ -270,37 +504,6 @@ export interface DesktopNimbusExperiment { featureValidationOptOut?: boolean; localizations?: ExperimentLocalizations | null; } -export interface ExperimentBucketConfig { - randomizationUnit: RandomizationUnit; - /** - * Additional inputs to the hashing function. - */ - namespace: string; - /** - * Index of the starting bucket of the range. - */ - start: number; - /** - * Number of buckets in the range. - */ - count: number; - /** - * The total number of buckets. - * - * You can assume this will always be 10000 - */ - total: number; -} -export interface ExperimentOutcome { - /** - * Identifier for the outcome. - */ - slug: string; - /** - * e.g., "primary" or "secondary". - */ - priority: string; -} /** * The branch definition supported on Firefox Desktop 95+. */ @@ -319,75 +522,11 @@ export interface DesktopExperimentBranch { * An array of feature configurations. */ features: ExperimentFeatureConfig[]; - feature: DesktopTombstoneFeatureConfig; /** * An optional string containing the title of the branch */ firefoxLabsTitle?: string; } -export interface ExperimentFeatureConfig { - /** - * The identifier for the feature flag. - */ - featureId: string; - /** - * The values that define the feature configuration. - * - * This should be validated against a schema. - */ - value: { - [k: string]: unknown; - }; -} -export interface DesktopTombstoneFeatureConfig { - featureId: "unused-feature-id-for-legacy-support"; - value: { - [k: string]: unknown; - }; - enabled: false; -} -/** - * Per-locale localization substitutions. - * - * The top level key is the locale (e.g., "en-US" or "fr"). Each entry is a mapping of - * string IDs to their localized equivalents. - */ -export interface ExperimentLocalizations { - [k: string]: { - [k: string]: string; - }; -} -/** - * A Nimbus experiment for Nimbus SDK-based applications. - */ -export interface SdkNimbusExperiment { - /** - * Branch configuration for the experiment. - */ - branches: SdkExperimentBranch[]; -} -/** - * The branch definition for SDK-based applications - * - * Supported on Firefox for Android 96+ and Firefox for iOS 39+ and all versions of - * Cirrus. - */ -export interface SdkExperimentBranch { - /** - * Identifier for the branch. - */ - slug: string; - /** - * Relative ratio of population for the branch. - * - * e.g., if branch A=1 and branch B=3, then branch A would get 25% of the population. - */ - ratio: number; - /** - * An array of feature configurations. - */ - features: ExperimentFeatureConfig[]; -} /** * The SDK-specific feature manifest. */ @@ -435,6 +574,37 @@ export interface SdkFeatureVariable { */ enum?: string[]; } +/** + * A Nimbus experiment for Nimbus SDK-based applications. + */ +export interface SdkNimbusExperiment { + /** + * Branch configuration for the experiment. + */ + branches: SdkExperimentBranch[]; +} +/** + * The branch definition for SDK-based applications + * + * Supported on Firefox for Android 96+ and Firefox for iOS 39+ and all versions of + * Cirrus. + */ +export interface SdkExperimentBranch { + /** + * Identifier for the branch. + */ + slug: string; + /** + * Relative ratio of population for the branch. + * + * e.g., if branch A=1 and branch B=3, then branch A would get 25% of the population. + */ + ratio: number; + /** + * An array of feature configurations. + */ + features: ExperimentFeatureConfig[]; +} export interface AnalysisError { analysis_basis?: AnalysisBasis | null; source?: LogSource | null; diff --git a/schemas/mozilla_nimbus_schemas/experiments/__init__.py b/schemas/mozilla_nimbus_schemas/experiments/__init__.py index c40000ad2..2bf014ca5 100644 --- a/schemas/mozilla_nimbus_schemas/experiments/__init__.py +++ b/schemas/mozilla_nimbus_schemas/experiments/__init__.py @@ -1,11 +1,17 @@ -from .experiments import DesktopNimbusExperiment, RandomizationUnit, SdkNimbusExperiment +from .experiments import ( + DesktopAllVersionsNimbusExperiment, + DesktopNimbusExperiment, + RandomizationUnit, + SdkNimbusExperiment, +) from .feature_manifests import DesktopFeature, DesktopFeatureManifest, SdkFeatureManifest __all__ = ( + "DesktopAllVersionsNimbusExperiment", "DesktopFeature", "DesktopFeatureManifest", "DesktopNimbusExperiment", - "SdkNimbusExperiment", "RandomizationUnit", "SdkFeatureManifest", + "SdkNimbusExperiment", ) diff --git a/schemas/mozilla_nimbus_schemas/experiments/experiments.py b/schemas/mozilla_nimbus_schemas/experiments/experiments.py index 1fd218593..d45c754b3 100644 --- a/schemas/mozilla_nimbus_schemas/experiments/experiments.py +++ b/schemas/mozilla_nimbus_schemas/experiments/experiments.py @@ -67,8 +67,8 @@ class BaseExperimentBranch(BaseModel): ) -class DesktopTombstoneFeatureConfig(ExperimentFeatureConfig): - featureId: Literal["unused-feature-id-for-legacy-support"] +class DesktopPre95FeatureConfig(ExperimentFeatureConfig): + featureId: Literal["this-is-included-for-desktop-pre-95-support"] enabled: Literal[False] value: dict[str, Any] @@ -76,15 +76,30 @@ class DesktopTombstoneFeatureConfig(ExperimentFeatureConfig): class DesktopExperimentBranch(BaseExperimentBranch): """The branch definition supported on Firefox Desktop 95+.""" - feature: DesktopTombstoneFeatureConfig = Field( + # Firefox Desktop-specific fields should be added to *this* schema. They will be + # inherited by the stricter DesktopAllVersionsExperimentBranch schema. + + firefoxLabsTitle: str | SkipJsonSchema[None] = Field( + description="An optional string containing the title of the branch", default=None + ) + + +class DesktopAllVersionsExperimentBranch(DesktopExperimentBranch): + """The branch definition supported on all Firefox Desktop versions. + + This version requires the feature field to be present to support older Firefox Desktop + clients. + """ + + # Firefox Desktop-specific fields should be added to DesktopExperimentBranch. They + # will be inherited by this schema. + + feature: DesktopPre95FeatureConfig = Field( description=( "The feature key must be provided with values to prevent crashes if the " "is encountered by Desktop clients earlier than version 95." ) ) - firefoxLabsTitle: str | SkipJsonSchema[None] = Field( - description="An optional string containing the title of the branch", default=None - ) class SdkExperimentBranch(BaseExperimentBranch): @@ -254,7 +269,14 @@ class BaseExperiment(BaseModel): class DesktopNimbusExperiment(BaseExperiment): - """A Nimbus experiment for Firefox Desktop.""" + """A Nimbus experiment for Firefox Desktop. + + This schema is less strict than DesktopAllVersionsNimbusExperiment and is intended for + use in Firefox Desktop. + """ + + # Firefox Desktop-specific fields should be added to *this* schema. They will be + # inherited by the stricter DesktopAllVersionsNimbusExperiment schema. branches: list[DesktopExperimentBranch] = Field( description="Branch configuration for the experiment." @@ -345,6 +367,22 @@ class DesktopNimbusExperiment(BaseExperiment): ) +class DesktopAllVersionsNimbusExperiment(DesktopNimbusExperiment): + """A Nimbus experiment for Firefox Desktop. + + This schema is more strict than DesktopNimbusExperiment and is backwards + comaptible with Firefox Desktop versions less than 95. It is intended for use inside + Experimenter itself. + """ + + # Firefox Desktop-specific fields should be added to DesktopNimbusExperiment. They + # will be inherited by this schema. + + branches: list[DesktopAllVersionsExperimentBranch] = Field( + description="Branch configuration for the experiment." + ) + + class SdkNimbusExperiment(BaseModel): """A Nimbus experiment for Nimbus SDK-based applications.""" diff --git a/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-107-featureValidationOptOut.json b/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-107-featureValidationOptOut.json index a0ebc2f90..4f9f24994 100644 --- a/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-107-featureValidationOptOut.json +++ b/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-107-featureValidationOptOut.json @@ -6,7 +6,7 @@ "branches": [ { "feature": { - "featureId": "unused-feature-id-for-legacy-support", + "featureId": "this-is-included-for-desktop-pre-95-support", "enabled": false, "value": {} }, @@ -27,7 +27,7 @@ }, { "feature": { - "featureId": "unused-feature-id-for-legacy-support", + "featureId": "this-is-included-for-desktop-pre-95-support", "enabled": false, "value": {} }, diff --git a/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-113-localizations.json b/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-113-localizations.json index 92a0686c5..2a466107f 100644 --- a/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-113-localizations.json +++ b/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-113-localizations.json @@ -6,7 +6,7 @@ "branches": [ { "feature": { - "featureId": "unused-feature-id-for-legacy-support", + "featureId": "this-is-included-for-desktop-pre-95-support", "enabled": false, "value": {} }, @@ -29,7 +29,7 @@ }, { "feature": { - "featureId": "unused-feature-id-for-legacy-support", + "featureId": "this-is-included-for-desktop-pre-95-support", "enabled": false, "value": {} }, diff --git a/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-115-locales.json b/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-115-locales.json index 6cf832580..238caeab5 100644 --- a/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-115-locales.json +++ b/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-115-locales.json @@ -6,7 +6,7 @@ "branches": [ { "feature": { - "featureId": "unused-feature-id-for-legacy-support", + "featureId": "this-is-included-for-desktop-pre-95-support", "enabled": false, "value": {} }, @@ -29,7 +29,7 @@ }, { "feature": { - "featureId": "unused-feature-id-for-legacy-support", + "featureId": "this-is-included-for-desktop-pre-95-support", "enabled": false, "value": {} }, diff --git a/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-123-publishedDate.json b/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-123-publishedDate.json index b7e1132cf..4f41d70f2 100644 --- a/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-123-publishedDate.json +++ b/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-123-publishedDate.json @@ -6,7 +6,7 @@ "branches": [ { "feature": { - "featureId": "unused-feature-id-for-legacy-support", + "featureId": "this-is-included-for-desktop-pre-95-support", "enabled": false, "value": {} }, @@ -27,7 +27,7 @@ }, { "feature": { - "featureId": "unused-feature-id-for-legacy-support", + "featureId": "this-is-included-for-desktop-pre-95-support", "enabled": false, "value": {} }, diff --git a/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-131-fxLabsOptIn-isNotRollout.json b/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-131-fxLabsOptIn-isNotRollout.json index 0891427d9..f44b0daad 100644 --- a/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-131-fxLabsOptIn-isNotRollout.json +++ b/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-131-fxLabsOptIn-isNotRollout.json @@ -7,7 +7,7 @@ { "firefoxLabsTitle": "branch-one-fx-labs_title", "feature": { - "featureId": "unused-feature-id-for-legacy-support", + "featureId": "this-is-included-for-desktop-pre-95-support", "enabled": false, "value": {} }, @@ -29,7 +29,7 @@ { "firefoxLabsTitle": "branch-one-fx-labs_title", "feature": { - "featureId": "unused-feature-id-for-legacy-support", + "featureId": "this-is-included-for-desktop-pre-95-support", "enabled": false, "value": {} }, diff --git a/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-131-fxLabsOptIn-isRollout.json b/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-131-fxLabsOptIn-isRollout.json index 059d573fc..cad993fc4 100644 --- a/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-131-fxLabsOptIn-isRollout.json +++ b/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-131-fxLabsOptIn-isRollout.json @@ -6,7 +6,7 @@ "branches": [ { "feature": { - "featureId": "unused-feature-id-for-legacy-support", + "featureId": "this-is-included-for-desktop-pre-95-support", "enabled": false, "value": {} }, @@ -27,7 +27,7 @@ }, { "feature": { - "featureId": "unused-feature-id-for-legacy-support", + "featureId": "this-is-included-for-desktop-pre-95-support", "enabled": false, "value": {} }, diff --git a/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-95-multifeature.json b/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-95-multifeature.json index 4b2d04c47..55888724b 100644 --- a/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-95-multifeature.json +++ b/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-95-multifeature.json @@ -6,7 +6,7 @@ "branches": [ { "feature": { - "featureId": "unused-feature-id-for-legacy-support", + "featureId": "this-is-included-for-desktop-pre-95-support", "enabled": false, "value": {} }, @@ -28,7 +28,7 @@ }, { "feature": { - "featureId": "unused-feature-id-for-legacy-support", + "featureId": "this-is-included-for-desktop-pre-95-support", "enabled": false, "value": {} }, diff --git a/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-98.json b/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-98.json index b7e1132cf..4f41d70f2 100644 --- a/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-98.json +++ b/schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-98.json @@ -6,7 +6,7 @@ "branches": [ { "feature": { - "featureId": "unused-feature-id-for-legacy-support", + "featureId": "this-is-included-for-desktop-pre-95-support", "enabled": false, "value": {} }, @@ -27,7 +27,7 @@ }, { "feature": { - "featureId": "unused-feature-id-for-legacy-support", + "featureId": "this-is-included-for-desktop-pre-95-support", "enabled": false, "value": {} }, diff --git a/schemas/mozilla_nimbus_schemas/tests/experiments/test_experiments.py b/schemas/mozilla_nimbus_schemas/tests/experiments/test_experiments.py index cdefa27cb..0d7abb922 100644 --- a/schemas/mozilla_nimbus_schemas/tests/experiments/test_experiments.py +++ b/schemas/mozilla_nimbus_schemas/tests/experiments/test_experiments.py @@ -2,13 +2,16 @@ import importlib.resources import json from functools import cache from pathlib import Path -from typing import Any +from typing import Any, Protocol +import jsonschema +import pydantic import pytest from jsonschema.protocols import Validator from jsonschema.validators import validator_for from mozilla_nimbus_schemas.experiments import ( + DesktopAllVersionsNimbusExperiment, DesktopNimbusExperiment, SdkNimbusExperiment, ) @@ -18,6 +21,12 @@ PACKAGE_DIR = importlib.resources.files("mozilla_nimbus_schemas") SCHEMAS_DIR = PACKAGE_DIR / "schemas" +@pytest.fixture +@cache +def desktop_all_versions_nimbus_experiment_schema_validator() -> Validator: + return load_schema("DesktopAllVersionsNimbusExperiment.schema.json") + + @pytest.fixture @cache def desktop_nimbus_experiment_schema_validator() -> Validator: @@ -30,6 +39,56 @@ def sdk_nimbus_experiment_schema_validator() -> Validator: return load_schema("SdkNimbusExperiment.schema.json") +class DesktopExperimentValidator(Protocol): + def __call__( + self, + experiment_json: dict[str, Any], + *, + valid: bool = True, + valid_all_versions: bool = True, + ): ... + + +@pytest.fixture +def validate_desktop_experiment( + desktop_nimbus_experiment_schema_validator, + desktop_all_versions_nimbus_experiment_schema_validator, +) -> DesktopExperimentValidator: + def _validate( + experiment_json: dict[str, Any], + *, + valid: bool = True, + valid_all_versions: bool = True, + ): + assert not (not valid and valid_all_versions), "valid_all_versions implies valid" + + if valid: + DesktopNimbusExperiment.model_validate(experiment_json) + desktop_nimbus_experiment_schema_validator.validate(experiment_json) + else: + with pytest.raises(pydantic.ValidationError): + DesktopNimbusExperiment.model_validate(experiment_json) + + with pytest.raises(jsonschema.ValidationError): + desktop_nimbus_experiment_schema_validator.validate(experiment_json) + + if valid_all_versions: + DesktopAllVersionsNimbusExperiment.model_validate(experiment_json) + desktop_all_versions_nimbus_experiment_schema_validator.validate( + experiment_json + ) + else: + with pytest.raises(pydantic.ValidationError): + DesktopAllVersionsNimbusExperiment.model_validate(experiment_json) + + with pytest.raises(jsonschema.ValidationError): + desktop_all_versions_nimbus_experiment_schema_validator.validate( + experiment_json + ) + + return _validate + + def load_schema(name: str) -> Validator: with SCHEMAS_DIR.joinpath(name).open() as f: schema = json.load(f) @@ -42,14 +101,20 @@ def load_schema(name: str) -> Validator: @pytest.mark.parametrize("experiment_file", FIXTURE_DIR.joinpath("desktop").iterdir()) def test_desktop_experiment_fixtures_are_valid( - experiment_file, desktop_nimbus_experiment_schema_validator + experiment_file, + validate_desktop_experiment, ): with open(experiment_file, "r") as f: experiment_json = json.load(f) - print(experiment_json) - DesktopNimbusExperiment.model_validate(experiment_json) - desktop_nimbus_experiment_schema_validator.validate(experiment_json) + validate_desktop_experiment(experiment_json) + + for branch in experiment_json["branches"]: + del branch["feature"] + + # Assert that this no longer passes with the strict schema, but passes with the + # regular schema. + validate_desktop_experiment(experiment_json, valid_all_versions=False) @pytest.mark.parametrize("experiment_file", FIXTURE_DIR.joinpath("sdk").iterdir()) @@ -58,44 +123,48 @@ def test_sdk_experiment_fixtures_are_valid( ): with open(experiment_file, "r") as f: experiment_json = json.load(f) - print(experiment_json) - SdkNimbusExperiment.model_validate(experiment_json) + SdkNimbusExperiment.model_validate(experiment_json) sdk_nimbus_experiment_schema_validator.validate(experiment_json) def test_desktop_nimbus_expirement_with_fxlabs_opt_in_is_not_rollout( - desktop_nimbus_experiment_schema_validator, + validate_desktop_experiment, ): - experiment = _desktop_nimbus_experiment_with_fxlabs_opt_in(isRollout=False) - - assert desktop_nimbus_experiment_schema_validator.is_valid(experiment) + experiment_json = _desktop_nimbus_experiment_with_fxlabs_opt_in(isRollout=False) + validate_desktop_experiment(experiment_json) def test_desktop_nimbus_experiment_with_fxlabs_opt_in_is_rollout( - desktop_nimbus_experiment_schema_validator, + validate_desktop_experiment, ): - experiment = _desktop_nimbus_experiment_with_fxlabs_opt_in(isRollout=True) - - assert desktop_nimbus_experiment_schema_validator.is_valid(experiment) + experiment_json = _desktop_nimbus_experiment_with_fxlabs_opt_in(isRollout=True) + validate_desktop_experiment(experiment_json) -def test_desktop_nimbus_experiment_without_fxlabs_opt_in( - desktop_nimbus_experiment_schema_validator, -): - experiment = _desktop_nimbus_experiment_without_fxlabs_opt_in() - - assert desktop_nimbus_experiment_schema_validator.is_valid(experiment) +def test_desktop_nimbus_experiment_without_fxlabs_opt_in(validate_desktop_experiment): + experiment_json = _desktop_nimbus_experiment_without_fxlabs_opt_in() + validate_desktop_experiment(experiment_json) def test_desktop_nimbus_experiment_with_fxlabs_opt_in_but_missing_required_fields( - desktop_nimbus_experiment_schema_validator, + validate_desktop_experiment, + desktop_all_versions_nimbus_experiment_schema_validator, ): - experiment = _desktop_nimbus_experiment_with_fxlabs_opt_in_missing_required_fields() + experiment_json = ( + _desktop_nimbus_experiment_with_fxlabs_opt_in_missing_required_fields() + ) + validate_desktop_experiment(experiment_json, valid=False, valid_all_versions=False) - assert not desktop_nimbus_experiment_schema_validator.is_valid(experiment) + assert not desktop_all_versions_nimbus_experiment_schema_validator.is_valid( + experiment_json + ) - errors = list(desktop_nimbus_experiment_schema_validator.iter_errors(experiment)) + errors = list( + desktop_all_versions_nimbus_experiment_schema_validator.iter_errors( + experiment_json + ) + ) error_messages = [e.message for e in errors] assert len(error_messages) == 4 @@ -112,7 +181,7 @@ def _desktop_nimbus_experiment(isRollout: bool) -> dict[str, Any]: "branches": [ { "feature": { - "featureId": "unused-feature-id-for-legacy-support", + "featureId": "this-is-included-for-desktop-pre-95-support", "enabled": False, "value": {}, }, @@ -125,7 +194,7 @@ def _desktop_nimbus_experiment(isRollout: bool) -> dict[str, Any]: }, { "feature": { - "featureId": "unused-feature-id-for-legacy-support", + "featureId": "this-is-included-for-desktop-pre-95-support", "enabled": False, "value": {}, }, diff --git a/schemas/package.json b/schemas/package.json index e33bd788d..4311762b2 100644 --- a/schemas/package.json +++ b/schemas/package.json @@ -1,6 +1,6 @@ { "name": "@mozilla/nimbus-schemas", - "version": "2024.11.3", + "version": "2024.11.4", "description": "Schemas used by Mozilla Nimbus and related projects.", "main": "index.d.ts", "repository": { diff --git a/schemas/pyproject.toml b/schemas/pyproject.toml index ac415d24d..86093898f 100644 --- a/schemas/pyproject.toml +++ b/schemas/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "mozilla-nimbus-schemas" -version = "2024.11.3" +version = "2024.11.4" description = "Schemas used by Mozilla Nimbus and related projects." authors = ["mikewilli"] license = "MPL 2.0" diff --git a/schemas/schemas/DesktopAllVersionsNimbusExperiment.schema.json b/schemas/schemas/DesktopAllVersionsNimbusExperiment.schema.json new file mode 100644 index 000000000..560cfdb38 --- /dev/null +++ b/schemas/schemas/DesktopAllVersionsNimbusExperiment.schema.json @@ -0,0 +1,397 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09/schema", + "title": "DesktopAllVersionsNimbusExperiment", + "description": "A Nimbus experiment for Firefox Desktop. This schema is more strict than DesktopNimbusExperiment and is backwards comaptible with Firefox Desktop versions less than 95. It is intended for use inside Experimenter itself.", + "type": "object", + "properties": { + "schemaVersion": { + "description": "Version of the NimbusExperiment schema this experiment refers to", + "type": "string" + }, + "slug": { + "description": "Unique identifier for the experiment", + "type": "string" + }, + "id": { + "description": "Unique identifier for the experiiment. This is a duplicate of slug, but is required field for all Remote Settings records.", + "type": "string" + }, + "appName": { + "description": "A slug identifying the targeted product of this experiment. It should be a lowercased_with_underscores name that is short and unambiguous and it should match the app_name found in https://probeinfo.telemetry.mozilla.org/glean/repositories. Examples are \"fenix\" and \"firefox_desktop\".", + "type": "string" + }, + "appId": { + "description": "The platform identifier for the targeted app. This should match app's identifier exactly as it appears in the relevant app store listing (for relevant platforms) or the app's Glean initialization (for other platforms). Examples are \"org.mozilla.firefox_beta\" and \"firefox-desktop\".", + "type": "string" + }, + "channel": { + "description": "A specific channel of an application such as \"nightly\", \"beta\", or \"release\".", + "type": "string" + }, + "userFacingName": { + "description": "Public name of the experiment that will be displayed on \"about:studies\".", + "type": "string" + }, + "userFacingDescription": { + "description": "Short public description of the experiment that will be displayed on \"about:studies\".", + "type": "string" + }, + "isEnrollmentPaused": { + "description": "When this property is set to true, the SDK should not enroll new users into the experiment that have not already been enrolled.", + "type": "boolean" + }, + "isRollout": { + "description": "When this property is set to true, treat this experiment as a rollout. Rollouts are currently handled as single-branch experiments separated from the bucketing namespace for normal experiments. See-also: https://mozilla-hub.atlassian.net/browse/SDK-405", + "type": "boolean" + }, + "bucketConfig": { + "$ref": "#/$defs/ExperimentBucketConfig", + "description": "Bucketing configuration." + }, + "outcomes": { + "description": "A list of outcomes relevant to the experiment analysis.", + "items": { + "$ref": "#/$defs/ExperimentOutcome" + }, + "type": "array" + }, + "featureIds": { + "description": "A list of featureIds the experiment contains configurations for.", + "items": { + "type": "string" + }, + "type": "array" + }, + "targeting": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "A JEXL targeting expression used to filter out experiments." + }, + "startDate": { + "anyOf": [ + { + "format": "date", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Actual publish date of the experiment. Note that this value is expected to be null in Remote Settings." + }, + "enrollmentEndDate": { + "anyOf": [ + { + "format": "date", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Actual enrollment end date of the experiment. Note that this value is expected to be null in Remote Settings." + }, + "endDate": { + "anyOf": [ + { + "format": "date", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Actual end date of this experiment. Note that this field is expected to be null in Remote Settings." + }, + "proposedDuration": { + "description": "Duration of the experiment from the start date in days. Note that this property is only used during the analysis phase (i.e., not by the SDK).", + "type": "integer" + }, + "proposedEnrollment": { + "description": "This represents the number of days that we expect to enroll new users. Note that this property is only used during the analysis phase (i.e., not by the SDK).", + "type": "integer" + }, + "referenceBranch": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The slug of the reference branch (i.e., the branch we consider \"control\")." + }, + "locales": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The list of locale codes (e.g., \"en-US\" or \"fr\") that this experiment is targeting. If null, all locales are targeted." + }, + "publishedDate": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The date that this experiment was first published to Remote Settings. If null, it has not yet been published." + }, + "branches": { + "description": "Branch configuration for the experiment.", + "items": { + "$ref": "#/$defs/DesktopAllVersionsExperimentBranch" + }, + "type": "array" + }, + "isFirefoxLabsOptIn": { + "description": "When this property is set to true, treat this experiment as aFirefox Labs experiment", + "type": "boolean" + }, + "firefoxLabsTitle": { + "description": "An optional string containing the Fluent ID for the title of the opt-in", + "type": "string" + }, + "firefoxLabsDescription": { + "description": "An optional string containing the Fluent ID for the description of the opt-in", + "type": "string" + }, + "featureValidationOptOut": { + "description": "Opt out of feature schema validation.", + "type": "boolean" + }, + "localizations": { + "anyOf": [ + { + "$ref": "#/$defs/ExperimentLocalizations" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "schemaVersion", + "slug", + "id", + "appName", + "appId", + "channel", + "userFacingName", + "userFacingDescription", + "isEnrollmentPaused", + "bucketConfig", + "startDate", + "endDate", + "proposedEnrollment", + "referenceBranch", + "branches" + ], + "dependentSchemas": { + "isFirefoxLabsOptIn": { + "if": { + "properties": { + "isFirefoxLabsOptIn": { + "const": true + } + } + }, + "then": { + "if": { + "properties": { + "isRollout": { + "const": false + } + }, + "required": [ + "isRollout" + ] + }, + "properties": { + "firefoxLabsTitle": { + "type": "string" + }, + "firefoxLabsDescription": { + "type": "string" + } + }, + "required": [ + "firefoxLabsTitle", + "firefoxLabsDescription" + ], + "then": { + "properties": { + "branches": { + "items": { + "required": [ + "firefoxLabsTitle" + ] + } + } + } + } + } + } + }, + "$defs": { + "DesktopAllVersionsExperimentBranch": { + "description": "The branch definition supported on all Firefox Desktop versions. This version requires the feature field to be present to support older Firefox Desktop clients.", + "properties": { + "slug": { + "description": "Identifier for the branch.", + "type": "string" + }, + "ratio": { + "description": "Relative ratio of population for the branch. e.g., if branch A=1 and branch B=3, then branch A would get 25% of the population.", + "type": "integer" + }, + "features": { + "description": "An array of feature configurations.", + "items": { + "$ref": "#/$defs/ExperimentFeatureConfig" + }, + "type": "array" + }, + "firefoxLabsTitle": { + "description": "An optional string containing the title of the branch", + "type": "string" + }, + "feature": { + "$ref": "#/$defs/DesktopPre95FeatureConfig", + "description": "The feature key must be provided with values to prevent crashes if the is encountered by Desktop clients earlier than version 95." + } + }, + "required": [ + "slug", + "ratio", + "features", + "feature" + ], + "type": "object" + }, + "DesktopPre95FeatureConfig": { + "properties": { + "featureId": { + "const": "this-is-included-for-desktop-pre-95-support", + "type": "string" + }, + "value": { + "type": "object" + }, + "enabled": { + "const": false, + "type": "boolean" + } + }, + "required": [ + "featureId", + "value", + "enabled" + ], + "type": "object" + }, + "ExperimentBucketConfig": { + "properties": { + "randomizationUnit": { + "$ref": "#/$defs/RandomizationUnit" + }, + "namespace": { + "description": "Additional inputs to the hashing function.", + "type": "string" + }, + "start": { + "description": "Index of the starting bucket of the range.", + "type": "integer" + }, + "count": { + "description": "Number of buckets in the range.", + "type": "integer" + }, + "total": { + "description": "The total number of buckets. You can assume this will always be 10000", + "type": "integer" + } + }, + "required": [ + "randomizationUnit", + "namespace", + "start", + "count", + "total" + ], + "type": "object" + }, + "ExperimentFeatureConfig": { + "properties": { + "featureId": { + "description": "The identifier for the feature flag.", + "type": "string" + }, + "value": { + "description": "The values that define the feature configuration. This should be validated against a schema.", + "type": "object" + } + }, + "required": [ + "featureId", + "value" + ], + "type": "object" + }, + "ExperimentLocalizations": { + "additionalProperties": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "description": "Per-locale localization substitutions. The top level key is the locale (e.g., \"en-US\" or \"fr\"). Each entry is a mapping of string IDs to their localized equivalents.", + "type": "object" + }, + "ExperimentOutcome": { + "properties": { + "slug": { + "description": "Identifier for the outcome.", + "type": "string" + }, + "priority": { + "description": "e.g., \"primary\" or \"secondary\".", + "type": "string" + } + }, + "required": [ + "slug", + "priority" + ], + "type": "object" + }, + "RandomizationUnit": { + "description": "A unique, stable indentifier for the user used as an input to bucket hashing.", + "enum": [ + "normandy_id", + "nimbus_id", + "user_id", + "group_id" + ], + "type": "string" + } + } +} diff --git a/schemas/schemas/DesktopNimbusExperiment.schema.json b/schemas/schemas/DesktopNimbusExperiment.schema.json index a15efa32e..eb447af58 100644 --- a/schemas/schemas/DesktopNimbusExperiment.schema.json +++ b/schemas/schemas/DesktopNimbusExperiment.schema.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft/2019-09/schema", "title": "DesktopNimbusExperiment", - "description": "A Nimbus experiment for Firefox Desktop.", + "description": "A Nimbus experiment for Firefox Desktop. This schema is less strict than DesktopAllVersionsNimbusExperiment and is intended for use in Firefox Desktop.", "type": "object", "properties": { "schemaVersion": { @@ -270,10 +270,6 @@ }, "type": "array" }, - "feature": { - "$ref": "#/$defs/DesktopTombstoneFeatureConfig", - "description": "The feature key must be provided with values to prevent crashes if the is encountered by Desktop clients earlier than version 95." - }, "firefoxLabsTitle": { "description": "An optional string containing the title of the branch", "type": "string" @@ -282,29 +278,7 @@ "required": [ "slug", "ratio", - "features", - "feature" - ], - "type": "object" - }, - "DesktopTombstoneFeatureConfig": { - "properties": { - "featureId": { - "const": "unused-feature-id-for-legacy-support", - "type": "string" - }, - "value": { - "type": "object" - }, - "enabled": { - "const": false, - "type": "boolean" - } - }, - "required": [ - "featureId", - "value", - "enabled" + "features" ], "type": "object" },