bug(schemas): Match the placeholder feature ID to Experimenter (#11718)

Because:

- Experimenter uses a different placeholder feature ID than the
specified in the schemas; and
- using the new split schema in Firefox Desktop would break the world

This commit:

- splits the Firefox Desktop schema into two versions: a strict,
backwards-compatible version (one that enforces the branches[].feature
field is present), and a more lax client version (that does not require
the branches[].feature field);
- updates the strict schema to use the correct placeholder feature ID;
- updates all the fixtures to use the correct placeholder;
- updates all the tests to test against both the strict and lax schemas;
and
- updates the schemas package to 2024.11.4.

Fixes #11717

schemas/VERSION

@@ -1 +1 @@
-2024.11.3
+2024.11.4

schemas/index.d.ts

@@ -6,13 +6,13 @@
  * make schemas_build
  */
 
-export type DesktopApplication = "firefox-desktop" | "firefox-desktop-background-task";
-export type FeatureVariableType = "int" | "string" | "boolean" | "json";
-export type PrefBranch = "default" | "user";
 /**
  * A unique, stable indentifier for the user used as an input to bucket hashing.
  */
 export type RandomizationUnit = "normandy_id" | "nimbus_id" | "user_id" | "group_id";
+export type DesktopApplication = "firefox-desktop" | "firefox-desktop-background-task";
+export type FeatureVariableType = "int" | "string" | "boolean" | "json";
+export type PrefBranch = "default" | "user";
 export type AnalysisBasis = "enrollments" | "exposures";
 export type LogSource = "jetstream" | "sizing" | "jetstream-preview";
 export type AnalysisErrors = AnalysisError[];
@@ -32,6 +32,237 @@ export type SizingMetricName = "active_hours" | "search_count" | "days_of_use" |
 export type StatisticIngestEnum = "percentage" | "binomial" | "mean" | "count";
 export type Statistics = Statistic[];
 
+/**
+ * A Nimbus experiment for Firefox Desktop.
+ *
+ * This schema is more strict than DesktopNimbusExperiment and is backwards
+ * comaptible with Firefox Desktop versions less than 95. It is intended for use inside
+ * Experimenter itself.
+ */
+export interface DesktopAllVersionsNimbusExperiment {
+  /**
+   * Version of the NimbusExperiment schema this experiment refers to
+   */
+  schemaVersion: string;
+  /**
+   * Unique identifier for the experiment
+   */
+  slug: string;
+  /**
+   * Unique identifier for the experiiment.
+   *
+   * This is a duplicate of slug, but is required field for all Remote Settings records.
+   */
+  id: string;
+  /**
+   * A slug identifying the targeted product of this experiment.
+   *
+   * It should be a lowercased_with_underscores name that is short and unambiguous and it should match the app_name found in https://probeinfo.telemetry.mozilla.org/glean/repositories. Examples are "fenix" and "firefox_desktop".
+   */
+  appName: string;
+  /**
+   * The platform identifier for the targeted app.
+   *
+   * This should match app's identifier exactly as it appears in the relevant app store listing (for relevant platforms) or the app's Glean initialization (for other platforms).
+   *
+   * Examples are "org.mozilla.firefox_beta" and "firefox-desktop".
+   */
+  appId: string;
+  /**
+   * A specific channel of an application such as "nightly", "beta", or "release".
+   */
+  channel: string;
+  /**
+   * Public name of the experiment that will be displayed on "about:studies".
+   */
+  userFacingName: string;
+  /**
+   * Short public description of the experiment that will be displayed on "about:studies".
+   */
+  userFacingDescription: string;
+  /**
+   * When this property is set to true, the SDK should not enroll new users into the experiment that have not already been enrolled.
+   */
+  isEnrollmentPaused: boolean;
+  /**
+   * When this property is set to true, treat this experiment as a rollout.
+   *
+   * Rollouts are currently handled as single-branch experiments separated from the bucketing namespace for normal experiments.
+   *
+   * See-also: https://mozilla-hub.atlassian.net/browse/SDK-405
+   */
+  isRollout?: boolean;
+  bucketConfig: ExperimentBucketConfig;
+  /**
+   * A list of outcomes relevant to the experiment analysis.
+   */
+  outcomes?: ExperimentOutcome[];
+  /**
+   * A list of featureIds the experiment contains configurations for.
+   */
+  featureIds?: string[];
+  /**
+   * A JEXL targeting expression used to filter out experiments.
+   */
+  targeting?: string | null;
+  /**
+   * Actual publish date of the experiment.
+   *
+   * Note that this value is expected to be null in Remote Settings.
+   */
+  startDate: string | null;
+  /**
+   * Actual enrollment end date of the experiment.
+   *
+   * Note that this value is expected to be null in Remote Settings.
+   */
+  enrollmentEndDate?: string | null;
+  /**
+   * Actual end date of this experiment.
+   *
+   * Note that this field is expected to be null in Remote Settings.
+   */
+  endDate: string | null;
+  /**
+   * Duration of the experiment from the start date in days.
+   *
+   * Note that this property is only used during the analysis phase (i.e., not by the SDK).
+   */
+  proposedDuration?: number;
+  /**
+   * This represents the number of days that we expect to enroll new users.
+   *
+   * Note that this property is only used during the analysis phase (i.e., not by the SDK).
+   */
+  proposedEnrollment: number;
+  /**
+   * The slug of the reference branch (i.e., the branch we consider "control").
+   */
+  referenceBranch: string | null;
+  /**
+   * The list of locale codes (e.g., "en-US" or "fr") that this experiment is targeting.
+   *
+   * If null, all locales are targeted.
+   */
+  locales?: string[] | null;
+  /**
+   * The date that this experiment was first published to Remote Settings.
+   *
+   * If null, it has not yet been published.
+   */
+  publishedDate?: string | null;
+  /**
+   * Branch configuration for the experiment.
+   */
+  branches: DesktopAllVersionsExperimentBranch[];
+  /**
+   * When this property is set to true, treat this experiment as aFirefox Labs experiment
+   */
+  isFirefoxLabsOptIn?: boolean;
+  /**
+   * An optional string containing the Fluent ID for the title of the opt-in
+   */
+  firefoxLabsTitle?: string;
+  /**
+   * An optional string containing the Fluent ID for the description of the opt-in
+   */
+  firefoxLabsDescription?: string;
+  /**
+   * Opt out of feature schema validation.
+   */
+  featureValidationOptOut?: boolean;
+  localizations?: ExperimentLocalizations | null;
+}
+export interface ExperimentBucketConfig {
+  randomizationUnit: RandomizationUnit;
+  /**
+   * Additional inputs to the hashing function.
+   */
+  namespace: string;
+  /**
+   * Index of the starting bucket of the range.
+   */
+  start: number;
+  /**
+   * Number of buckets in the range.
+   */
+  count: number;
+  /**
+   * The total number of buckets.
+   *
+   * You can assume this will always be 10000
+   */
+  total: number;
+}
+export interface ExperimentOutcome {
+  /**
+   * Identifier for the outcome.
+   */
+  slug: string;
+  /**
+   * e.g., "primary" or "secondary".
+   */
+  priority: string;
+}
+/**
+ * The branch definition supported on all Firefox Desktop versions.
+ *
+ * This version requires the feature field to be present to support older Firefox Desktop
+ * clients.
+ */
+export interface DesktopAllVersionsExperimentBranch {
+  /**
+   * Identifier for the branch.
+   */
+  slug: string;
+  /**
+   * Relative ratio of population for the branch.
+   *
+   * e.g., if branch A=1 and branch B=3, then branch A would get 25% of the population.
+   */
+  ratio: number;
+  /**
+   * An array of feature configurations.
+   */
+  features: ExperimentFeatureConfig[];
+  /**
+   * An optional string containing the title of the branch
+   */
+  firefoxLabsTitle?: string;
+  feature: DesktopPre95FeatureConfig;
+}
+export interface ExperimentFeatureConfig {
+  /**
+   * The identifier for the feature flag.
+   */
+  featureId: string;
+  /**
+   * The values that define the feature configuration.
+   *
+   * This should be validated against a schema.
+   */
+  value: {
+    [k: string]: unknown;
+  };
+}
+export interface DesktopPre95FeatureConfig {
+  featureId: "this-is-included-for-desktop-pre-95-support";
+  value: {
+    [k: string]: unknown;
+  };
+  enabled: false;
+}
+/**
+ * Per-locale localization substitutions.
+ *
+ * The top level key is the locale (e.g., "en-US" or "fr"). Each entry is a mapping of
+ * string IDs to their localized equivalents.
+ */
+export interface ExperimentLocalizations {
+  [k: string]: {
+    [k: string]: string;
+  };
+}
 /**
  * A feature.
  */
@@ -135,6 +366,9 @@ export interface DesktopFeatureManifest {
 }
 /**
  * A Nimbus experiment for Firefox Desktop.
+ *
+ * This schema is less strict than DesktopAllVersionsNimbusExperiment and is intended for
+ * use in Firefox Desktop.
  */
 export interface DesktopNimbusExperiment {
   /**
@@ -270,37 +504,6 @@ export interface DesktopNimbusExperiment {
   featureValidationOptOut?: boolean;
   localizations?: ExperimentLocalizations | null;
 }
-export interface ExperimentBucketConfig {
-  randomizationUnit: RandomizationUnit;
-  /**
-   * Additional inputs to the hashing function.
-   */
-  namespace: string;
-  /**
-   * Index of the starting bucket of the range.
-   */
-  start: number;
-  /**
-   * Number of buckets in the range.
-   */
-  count: number;
-  /**
-   * The total number of buckets.
-   *
-   * You can assume this will always be 10000
-   */
-  total: number;
-}
-export interface ExperimentOutcome {
-  /**
-   * Identifier for the outcome.
-   */
-  slug: string;
-  /**
-   * e.g., "primary" or "secondary".
-   */
-  priority: string;
-}
 /**
  * The branch definition supported on Firefox Desktop 95+.
  */
@@ -319,75 +522,11 @@ export interface DesktopExperimentBranch {
    * An array of feature configurations.
    */
   features: ExperimentFeatureConfig[];
-  feature: DesktopTombstoneFeatureConfig;
   /**
    * An optional string containing the title of the branch
    */
   firefoxLabsTitle?: string;
 }
-export interface ExperimentFeatureConfig {
-  /**
-   * The identifier for the feature flag.
-   */
-  featureId: string;
-  /**
-   * The values that define the feature configuration.
-   *
-   * This should be validated against a schema.
-   */
-  value: {
-    [k: string]: unknown;
-  };
-}
-export interface DesktopTombstoneFeatureConfig {
-  featureId: "unused-feature-id-for-legacy-support";
-  value: {
-    [k: string]: unknown;
-  };
-  enabled: false;
-}
-/**
- * Per-locale localization substitutions.
- *
- * The top level key is the locale (e.g., "en-US" or "fr"). Each entry is a mapping of
- * string IDs to their localized equivalents.
- */
-export interface ExperimentLocalizations {
-  [k: string]: {
-    [k: string]: string;
-  };
-}
-/**
- * A Nimbus experiment for Nimbus SDK-based applications.
- */
-export interface SdkNimbusExperiment {
-  /**
-   * Branch configuration for the experiment.
-   */
-  branches: SdkExperimentBranch[];
-}
-/**
- * The branch definition for SDK-based applications
- *
- * Supported on Firefox for Android 96+ and Firefox for iOS 39+ and all versions of
- * Cirrus.
- */
-export interface SdkExperimentBranch {
-  /**
-   * Identifier for the branch.
-   */
-  slug: string;
-  /**
-   * Relative ratio of population for the branch.
-   *
-   * e.g., if branch A=1 and branch B=3, then branch A would get 25% of the population.
-   */
-  ratio: number;
-  /**
-   * An array of feature configurations.
-   */
-  features: ExperimentFeatureConfig[];
-}
 /**
  * The SDK-specific feature manifest.
  */
@@ -435,6 +574,37 @@ export interface SdkFeatureVariable {
    */
   enum?: string[];
 }
+/**
+ * A Nimbus experiment for Nimbus SDK-based applications.
+ */
+export interface SdkNimbusExperiment {
+  /**
+   * Branch configuration for the experiment.
+   */
+  branches: SdkExperimentBranch[];
+}
+/**
+ * The branch definition for SDK-based applications
+ *
+ * Supported on Firefox for Android 96+ and Firefox for iOS 39+ and all versions of
+ * Cirrus.
+ */
+export interface SdkExperimentBranch {
+  /**
+   * Identifier for the branch.
+   */
+  slug: string;
+  /**
+   * Relative ratio of population for the branch.
+   *
+   * e.g., if branch A=1 and branch B=3, then branch A would get 25% of the population.
+   */
+  ratio: number;
+  /**
+   * An array of feature configurations.
+   */
+  features: ExperimentFeatureConfig[];
+}
 export interface AnalysisError {
   analysis_basis?: AnalysisBasis | null;
   source?: LogSource | null;

schemas/mozilla_nimbus_schemas/experiments/__init__.py

@@ -1,11 +1,17 @@
-from .experiments import DesktopNimbusExperiment, RandomizationUnit, SdkNimbusExperiment
+from .experiments import (
+    DesktopAllVersionsNimbusExperiment,
+    DesktopNimbusExperiment,
+    RandomizationUnit,
+    SdkNimbusExperiment,
+)
 from .feature_manifests import DesktopFeature, DesktopFeatureManifest, SdkFeatureManifest
 
 __all__ = (
+    "DesktopAllVersionsNimbusExperiment",
     "DesktopFeature",
     "DesktopFeatureManifest",
     "DesktopNimbusExperiment",
-    "SdkNimbusExperiment",
     "RandomizationUnit",
     "SdkFeatureManifest",
+    "SdkNimbusExperiment",
 )

schemas/mozilla_nimbus_schemas/experiments/experiments.py

@@ -67,8 +67,8 @@ class BaseExperimentBranch(BaseModel):
     )
 
 
-class DesktopTombstoneFeatureConfig(ExperimentFeatureConfig):
+class DesktopPre95FeatureConfig(ExperimentFeatureConfig):
-    featureId: Literal["unused-feature-id-for-legacy-support"]
+    featureId: Literal["this-is-included-for-desktop-pre-95-support"]
     enabled: Literal[False]
     value: dict[str, Any]
 
@@ -76,15 +76,30 @@ class DesktopTombstoneFeatureConfig(ExperimentFeatureConfig):
 class DesktopExperimentBranch(BaseExperimentBranch):
     """The branch definition supported on Firefox Desktop 95+."""
 
-    feature: DesktopTombstoneFeatureConfig = Field(
+    # Firefox Desktop-specific fields should be added to *this* schema. They will be
+    # inherited by the stricter DesktopAllVersionsExperimentBranch schema.
+
+    firefoxLabsTitle: str | SkipJsonSchema[None] = Field(
+        description="An optional string containing the title of the branch", default=None
+    )
+
+
+class DesktopAllVersionsExperimentBranch(DesktopExperimentBranch):
+    """The branch definition supported on all Firefox Desktop versions.
+
+    This version requires the feature field to be present to support older Firefox Desktop
+    clients.
+    """
+
+    # Firefox Desktop-specific fields should be added to DesktopExperimentBranch. They
+    # will be inherited by this schema.
+
+    feature: DesktopPre95FeatureConfig = Field(
         description=(
             "The feature key must be provided with values to prevent crashes if the "
             "is encountered by Desktop clients earlier than version 95."
         )
     )
-    firefoxLabsTitle: str | SkipJsonSchema[None] = Field(
-        description="An optional string containing the title of the branch", default=None
-    )
 
 
 class SdkExperimentBranch(BaseExperimentBranch):
@@ -254,7 +269,14 @@ class BaseExperiment(BaseModel):
 
 
 class DesktopNimbusExperiment(BaseExperiment):
-    """A Nimbus experiment for Firefox Desktop."""
+    """A Nimbus experiment for Firefox Desktop.
+
+    This schema is less strict than DesktopAllVersionsNimbusExperiment and is intended for
+    use in Firefox Desktop.
+    """
+
+    # Firefox Desktop-specific fields should be added to *this* schema. They will be
+    # inherited by the stricter DesktopAllVersionsNimbusExperiment schema.
 
     branches: list[DesktopExperimentBranch] = Field(
         description="Branch configuration for the experiment."
@@ -345,6 +367,22 @@ class DesktopNimbusExperiment(BaseExperiment):
     )
 
 
+class DesktopAllVersionsNimbusExperiment(DesktopNimbusExperiment):
+    """A Nimbus experiment for Firefox Desktop.
+
+    This schema is more strict than DesktopNimbusExperiment and is backwards
+    comaptible with Firefox Desktop versions less than 95. It is intended for use inside
+    Experimenter itself.
+    """
+
+    # Firefox Desktop-specific fields should be added to DesktopNimbusExperiment. They
+    # will be inherited by this schema.
+
+    branches: list[DesktopAllVersionsExperimentBranch] = Field(
+        description="Branch configuration for the experiment."
+    )
+
+
 class SdkNimbusExperiment(BaseModel):
     """A Nimbus experiment for Nimbus SDK-based applications."""
 

schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-107-featureValidationOptOut.json

@@ -6,7 +6,7 @@
   "branches": [
     {
       "feature": {
-        "featureId": "unused-feature-id-for-legacy-support",
+        "featureId": "this-is-included-for-desktop-pre-95-support",
         "enabled": false,
         "value": {}
       },
@@ -27,7 +27,7 @@
     },
     {
       "feature": {
-        "featureId": "unused-feature-id-for-legacy-support",
+        "featureId": "this-is-included-for-desktop-pre-95-support",
         "enabled": false,
         "value": {}
       },

schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-113-localizations.json

@@ -6,7 +6,7 @@
   "branches": [
     {
       "feature": {
-        "featureId": "unused-feature-id-for-legacy-support",
+        "featureId": "this-is-included-for-desktop-pre-95-support",
         "enabled": false,
         "value": {}
       },
@@ -29,7 +29,7 @@
     },
     {
       "feature": {
-        "featureId": "unused-feature-id-for-legacy-support",
+        "featureId": "this-is-included-for-desktop-pre-95-support",
         "enabled": false,
         "value": {}
       },

schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-115-locales.json

@@ -6,7 +6,7 @@
   "branches": [
     {
       "feature": {
-        "featureId": "unused-feature-id-for-legacy-support",
+        "featureId": "this-is-included-for-desktop-pre-95-support",
         "enabled": false,
         "value": {}
       },
@@ -29,7 +29,7 @@
     },
     {
       "feature": {
-        "featureId": "unused-feature-id-for-legacy-support",
+        "featureId": "this-is-included-for-desktop-pre-95-support",
         "enabled": false,
         "value": {}
       },

schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-123-publishedDate.json

@@ -6,7 +6,7 @@
   "branches": [
     {
       "feature": {
-        "featureId": "unused-feature-id-for-legacy-support",
+        "featureId": "this-is-included-for-desktop-pre-95-support",
         "enabled": false,
         "value": {}
       },
@@ -27,7 +27,7 @@
     },
     {
       "feature": {
-        "featureId": "unused-feature-id-for-legacy-support",
+        "featureId": "this-is-included-for-desktop-pre-95-support",
         "enabled": false,
         "value": {}
       },

schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-131-fxLabsOptIn-isNotRollout.json

@@ -7,7 +7,7 @@
     {
       "firefoxLabsTitle": "branch-one-fx-labs_title",
       "feature": {
-        "featureId": "unused-feature-id-for-legacy-support",
+        "featureId": "this-is-included-for-desktop-pre-95-support",
         "enabled": false,
         "value": {}
       },
@@ -29,7 +29,7 @@
     {
       "firefoxLabsTitle": "branch-one-fx-labs_title",
       "feature": {
-        "featureId": "unused-feature-id-for-legacy-support",
+        "featureId": "this-is-included-for-desktop-pre-95-support",
         "enabled": false,
         "value": {}
       },

schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-131-fxLabsOptIn-isRollout.json

@@ -6,7 +6,7 @@
   "branches": [
     {
       "feature": {
-        "featureId": "unused-feature-id-for-legacy-support",
+        "featureId": "this-is-included-for-desktop-pre-95-support",
         "enabled": false,
         "value": {}
       },
@@ -27,7 +27,7 @@
     },
     {
       "feature": {
-        "featureId": "unused-feature-id-for-legacy-support",
+        "featureId": "this-is-included-for-desktop-pre-95-support",
         "enabled": false,
         "value": {}
       },

schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-95-multifeature.json

@@ -6,7 +6,7 @@
   "branches": [
     {
       "feature": {
-        "featureId": "unused-feature-id-for-legacy-support",
+        "featureId": "this-is-included-for-desktop-pre-95-support",
         "enabled": false,
         "value": {}
       },
@@ -28,7 +28,7 @@
     },
     {
       "feature": {
-        "featureId": "unused-feature-id-for-legacy-support",
+        "featureId": "this-is-included-for-desktop-pre-95-support",
         "enabled": false,
         "value": {}
       },

schemas/mozilla_nimbus_schemas/tests/experiments/fixtures/experiments/desktop/desktop-98.json

@@ -6,7 +6,7 @@
   "branches": [
     {
       "feature": {
-        "featureId": "unused-feature-id-for-legacy-support",
+        "featureId": "this-is-included-for-desktop-pre-95-support",
         "enabled": false,
         "value": {}
       },
@@ -27,7 +27,7 @@
     },
     {
       "feature": {
-        "featureId": "unused-feature-id-for-legacy-support",
+        "featureId": "this-is-included-for-desktop-pre-95-support",
         "enabled": false,
         "value": {}
       },

schemas/mozilla_nimbus_schemas/tests/experiments/test_experiments.py

@@ -2,13 +2,16 @@ import importlib.resources
 import json
 from functools import cache
 from pathlib import Path
-from typing import Any
+from typing import Any, Protocol
 
+import jsonschema
+import pydantic
 import pytest
 from jsonschema.protocols import Validator
 from jsonschema.validators import validator_for
 
 from mozilla_nimbus_schemas.experiments import (
+    DesktopAllVersionsNimbusExperiment,
     DesktopNimbusExperiment,
     SdkNimbusExperiment,
 )
@@ -18,6 +21,12 @@ PACKAGE_DIR = importlib.resources.files("mozilla_nimbus_schemas")
 SCHEMAS_DIR = PACKAGE_DIR / "schemas"
 
 
+@pytest.fixture
+@cache
+def desktop_all_versions_nimbus_experiment_schema_validator() -> Validator:
+    return load_schema("DesktopAllVersionsNimbusExperiment.schema.json")
+
+
 @pytest.fixture
 @cache
 def desktop_nimbus_experiment_schema_validator() -> Validator:
@@ -30,6 +39,56 @@ def sdk_nimbus_experiment_schema_validator() -> Validator:
     return load_schema("SdkNimbusExperiment.schema.json")
 
 
+class DesktopExperimentValidator(Protocol):
+    def __call__(
+        self,
+        experiment_json: dict[str, Any],
+        *,
+        valid: bool = True,
+        valid_all_versions: bool = True,
+    ): ...
+
+
+@pytest.fixture
+def validate_desktop_experiment(
+    desktop_nimbus_experiment_schema_validator,
+    desktop_all_versions_nimbus_experiment_schema_validator,
+) -> DesktopExperimentValidator:
+    def _validate(
+        experiment_json: dict[str, Any],
+        *,
+        valid: bool = True,
+        valid_all_versions: bool = True,
+    ):
+        assert not (not valid and valid_all_versions), "valid_all_versions implies valid"
+
+        if valid:
+            DesktopNimbusExperiment.model_validate(experiment_json)
+            desktop_nimbus_experiment_schema_validator.validate(experiment_json)
+        else:
+            with pytest.raises(pydantic.ValidationError):
+                DesktopNimbusExperiment.model_validate(experiment_json)
+
+            with pytest.raises(jsonschema.ValidationError):
+                desktop_nimbus_experiment_schema_validator.validate(experiment_json)
+
+        if valid_all_versions:
+            DesktopAllVersionsNimbusExperiment.model_validate(experiment_json)
+            desktop_all_versions_nimbus_experiment_schema_validator.validate(
+                experiment_json
+            )
+        else:
+            with pytest.raises(pydantic.ValidationError):
+                DesktopAllVersionsNimbusExperiment.model_validate(experiment_json)
+
+            with pytest.raises(jsonschema.ValidationError):
+                desktop_all_versions_nimbus_experiment_schema_validator.validate(
+                    experiment_json
+                )
+
+    return _validate
+
+
 def load_schema(name: str) -> Validator:
     with SCHEMAS_DIR.joinpath(name).open() as f:
         schema = json.load(f)
@@ -42,14 +101,20 @@ def load_schema(name: str) -> Validator:
 
 @pytest.mark.parametrize("experiment_file", FIXTURE_DIR.joinpath("desktop").iterdir())
 def test_desktop_experiment_fixtures_are_valid(
-    experiment_file, desktop_nimbus_experiment_schema_validator
+    experiment_file,
+    validate_desktop_experiment,
 ):
     with open(experiment_file, "r") as f:
         experiment_json = json.load(f)
-        print(experiment_json)
-        DesktopNimbusExperiment.model_validate(experiment_json)
 
-    desktop_nimbus_experiment_schema_validator.validate(experiment_json)
+    validate_desktop_experiment(experiment_json)
+
+    for branch in experiment_json["branches"]:
+        del branch["feature"]
+
+    # Assert that this no longer passes with the strict schema, but passes with the
+    # regular schema.
+    validate_desktop_experiment(experiment_json, valid_all_versions=False)
 
 
 @pytest.mark.parametrize("experiment_file", FIXTURE_DIR.joinpath("sdk").iterdir())
@@ -58,44 +123,48 @@ def test_sdk_experiment_fixtures_are_valid(
 ):
     with open(experiment_file, "r") as f:
         experiment_json = json.load(f)
-        print(experiment_json)
-        SdkNimbusExperiment.model_validate(experiment_json)
 
+    SdkNimbusExperiment.model_validate(experiment_json)
     sdk_nimbus_experiment_schema_validator.validate(experiment_json)
 
 
 def test_desktop_nimbus_expirement_with_fxlabs_opt_in_is_not_rollout(
-    desktop_nimbus_experiment_schema_validator,
+    validate_desktop_experiment,
 ):
-    experiment = _desktop_nimbus_experiment_with_fxlabs_opt_in(isRollout=False)
+    experiment_json = _desktop_nimbus_experiment_with_fxlabs_opt_in(isRollout=False)
-
+    validate_desktop_experiment(experiment_json)
-    assert desktop_nimbus_experiment_schema_validator.is_valid(experiment)
 
 
 def test_desktop_nimbus_experiment_with_fxlabs_opt_in_is_rollout(
-    desktop_nimbus_experiment_schema_validator,
+    validate_desktop_experiment,
 ):
-    experiment = _desktop_nimbus_experiment_with_fxlabs_opt_in(isRollout=True)
+    experiment_json = _desktop_nimbus_experiment_with_fxlabs_opt_in(isRollout=True)
-
+    validate_desktop_experiment(experiment_json)
-    assert desktop_nimbus_experiment_schema_validator.is_valid(experiment)
 
 
-def test_desktop_nimbus_experiment_without_fxlabs_opt_in(
+def test_desktop_nimbus_experiment_without_fxlabs_opt_in(validate_desktop_experiment):
-    desktop_nimbus_experiment_schema_validator,
+    experiment_json = _desktop_nimbus_experiment_without_fxlabs_opt_in()
-):
+    validate_desktop_experiment(experiment_json)
-    experiment = _desktop_nimbus_experiment_without_fxlabs_opt_in()
-
-    assert desktop_nimbus_experiment_schema_validator.is_valid(experiment)
 
 
 def test_desktop_nimbus_experiment_with_fxlabs_opt_in_but_missing_required_fields(
-    desktop_nimbus_experiment_schema_validator,
+    validate_desktop_experiment,
+    desktop_all_versions_nimbus_experiment_schema_validator,
 ):
-    experiment = _desktop_nimbus_experiment_with_fxlabs_opt_in_missing_required_fields()
+    experiment_json = (
+        _desktop_nimbus_experiment_with_fxlabs_opt_in_missing_required_fields()
+    )
+    validate_desktop_experiment(experiment_json, valid=False, valid_all_versions=False)
 
-    assert not desktop_nimbus_experiment_schema_validator.is_valid(experiment)
+    assert not desktop_all_versions_nimbus_experiment_schema_validator.is_valid(
+        experiment_json
+    )
 
-    errors = list(desktop_nimbus_experiment_schema_validator.iter_errors(experiment))
+    errors = list(
+        desktop_all_versions_nimbus_experiment_schema_validator.iter_errors(
+            experiment_json
+        )
+    )
     error_messages = [e.message for e in errors]
 
     assert len(error_messages) == 4
@@ -112,7 +181,7 @@ def _desktop_nimbus_experiment(isRollout: bool) -> dict[str, Any]:
         "branches": [
             {
                 "feature": {
-                    "featureId": "unused-feature-id-for-legacy-support",
+                    "featureId": "this-is-included-for-desktop-pre-95-support",
                     "enabled": False,
                     "value": {},
                 },
@@ -125,7 +194,7 @@ def _desktop_nimbus_experiment(isRollout: bool) -> dict[str, Any]:
             },
             {
                 "feature": {
-                    "featureId": "unused-feature-id-for-legacy-support",
+                    "featureId": "this-is-included-for-desktop-pre-95-support",
                     "enabled": False,
                     "value": {},
                 },

schemas/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mozilla/nimbus-schemas",
-  "version": "2024.11.3",
+  "version": "2024.11.4",
   "description": "Schemas used by Mozilla Nimbus and related projects.",
   "main": "index.d.ts",
   "repository": {

schemas/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "mozilla-nimbus-schemas"
-version = "2024.11.3"
+version = "2024.11.4"
 description = "Schemas used by Mozilla Nimbus and related projects."
 authors = ["mikewilli"]
 license = "MPL 2.0"

schemas/schemas/DesktopAllVersionsNimbusExperiment.schema.json

@@ -0,0 +1,397 @@
+{
+  "$schema": "https://json-schema.org/draft/2019-09/schema",
+  "title": "DesktopAllVersionsNimbusExperiment",
+  "description": "A Nimbus experiment for Firefox Desktop. This schema is more strict than DesktopNimbusExperiment and is backwards comaptible with Firefox Desktop versions less than 95. It is intended for use inside Experimenter itself.",
+  "type": "object",
+  "properties": {
+    "schemaVersion": {
+      "description": "Version of the NimbusExperiment schema this experiment refers to",
+      "type": "string"
+    },
+    "slug": {
+      "description": "Unique identifier for the experiment",
+      "type": "string"
+    },
+    "id": {
+      "description": "Unique identifier for the experiiment. This is a duplicate of slug, but is required field for all Remote Settings records.",
+      "type": "string"
+    },
+    "appName": {
+      "description": "A slug identifying the targeted product of this experiment. It should be a lowercased_with_underscores name that is short and unambiguous and it should match the app_name found in https://probeinfo.telemetry.mozilla.org/glean/repositories. Examples are \"fenix\" and \"firefox_desktop\".",
+      "type": "string"
+    },
+    "appId": {
+      "description": "The platform identifier for the targeted app. This should match app's identifier exactly as it appears in the relevant app store listing (for relevant platforms) or the app's Glean initialization (for other platforms). Examples are \"org.mozilla.firefox_beta\" and \"firefox-desktop\".",
+      "type": "string"
+    },
+    "channel": {
+      "description": "A specific channel of an application such as \"nightly\", \"beta\", or \"release\".",
+      "type": "string"
+    },
+    "userFacingName": {
+      "description": "Public name of the experiment that will be displayed on \"about:studies\".",
+      "type": "string"
+    },
+    "userFacingDescription": {
+      "description": "Short public description of the experiment that will be displayed on \"about:studies\".",
+      "type": "string"
+    },
+    "isEnrollmentPaused": {
+      "description": "When this property is set to true, the SDK should not enroll new users into the experiment that have not already been enrolled.",
+      "type": "boolean"
+    },
+    "isRollout": {
+      "description": "When this property is set to true, treat this experiment as a rollout. Rollouts are currently handled as single-branch experiments separated from the bucketing namespace for normal experiments. See-also: https://mozilla-hub.atlassian.net/browse/SDK-405",
+      "type": "boolean"
+    },
+    "bucketConfig": {
+      "$ref": "#/$defs/ExperimentBucketConfig",
+      "description": "Bucketing configuration."
+    },
+    "outcomes": {
+      "description": "A list of outcomes relevant to the experiment analysis.",
+      "items": {
+        "$ref": "#/$defs/ExperimentOutcome"
+      },
+      "type": "array"
+    },
+    "featureIds": {
+      "description": "A list of featureIds the experiment contains configurations for.",
+      "items": {
+        "type": "string"
+      },
+      "type": "array"
+    },
+    "targeting": {
+      "anyOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "description": "A JEXL targeting expression used to filter out experiments."
+    },
+    "startDate": {
+      "anyOf": [
+        {
+          "format": "date",
+          "type": "string"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "description": "Actual publish date of the experiment. Note that this value is expected to be null in Remote Settings."
+    },
+    "enrollmentEndDate": {
+      "anyOf": [
+        {
+          "format": "date",
+          "type": "string"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "description": "Actual enrollment end date of the experiment. Note that this value is expected to be null in Remote Settings."
+    },
+    "endDate": {
+      "anyOf": [
+        {
+          "format": "date",
+          "type": "string"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "description": "Actual end date of this experiment. Note that this field is expected to be null in Remote Settings."
+    },
+    "proposedDuration": {
+      "description": "Duration of the experiment from the start date in days. Note that this property is only used during the analysis phase (i.e., not by the SDK).",
+      "type": "integer"
+    },
+    "proposedEnrollment": {
+      "description": "This represents the number of days that we expect to enroll new users. Note that this property is only used during the analysis phase (i.e., not by the SDK).",
+      "type": "integer"
+    },
+    "referenceBranch": {
+      "anyOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "description": "The slug of the reference branch (i.e., the branch we consider \"control\")."
+    },
+    "locales": {
+      "anyOf": [
+        {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "description": "The list of locale codes (e.g., \"en-US\" or \"fr\") that this experiment is targeting. If null, all locales are targeted."
+    },
+    "publishedDate": {
+      "anyOf": [
+        {
+          "format": "date-time",
+          "type": "string"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "description": "The date that this experiment was first published to Remote Settings. If null, it has not yet been published."
+    },
+    "branches": {
+      "description": "Branch configuration for the experiment.",
+      "items": {
+        "$ref": "#/$defs/DesktopAllVersionsExperimentBranch"
+      },
+      "type": "array"
+    },
+    "isFirefoxLabsOptIn": {
+      "description": "When this property is set to true, treat this experiment as aFirefox Labs experiment",
+      "type": "boolean"
+    },
+    "firefoxLabsTitle": {
+      "description": "An optional string containing the Fluent ID for the title of the opt-in",
+      "type": "string"
+    },
+    "firefoxLabsDescription": {
+      "description": "An optional string containing the Fluent ID for the description of the opt-in",
+      "type": "string"
+    },
+    "featureValidationOptOut": {
+      "description": "Opt out of feature schema validation.",
+      "type": "boolean"
+    },
+    "localizations": {
+      "anyOf": [
+        {
+          "$ref": "#/$defs/ExperimentLocalizations"
+        },
+        {
+          "type": "null"
+        }
+      ]
+    }
+  },
+  "required": [
+    "schemaVersion",
+    "slug",
+    "id",
+    "appName",
+    "appId",
+    "channel",
+    "userFacingName",
+    "userFacingDescription",
+    "isEnrollmentPaused",
+    "bucketConfig",
+    "startDate",
+    "endDate",
+    "proposedEnrollment",
+    "referenceBranch",
+    "branches"
+  ],
+  "dependentSchemas": {
+    "isFirefoxLabsOptIn": {
+      "if": {
+        "properties": {
+          "isFirefoxLabsOptIn": {
+            "const": true
+          }
+        }
+      },
+      "then": {
+        "if": {
+          "properties": {
+            "isRollout": {
+              "const": false
+            }
+          },
+          "required": [
+            "isRollout"
+          ]
+        },
+        "properties": {
+          "firefoxLabsTitle": {
+            "type": "string"
+          },
+          "firefoxLabsDescription": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "firefoxLabsTitle",
+          "firefoxLabsDescription"
+        ],
+        "then": {
+          "properties": {
+            "branches": {
+              "items": {
+                "required": [
+                  "firefoxLabsTitle"
+                ]
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "$defs": {
+    "DesktopAllVersionsExperimentBranch": {
+      "description": "The branch definition supported on all Firefox Desktop versions. This version requires the feature field to be present to support older Firefox Desktop clients.",
+      "properties": {
+        "slug": {
+          "description": "Identifier for the branch.",
+          "type": "string"
+        },
+        "ratio": {
+          "description": "Relative ratio of population for the branch. e.g., if branch A=1 and branch B=3, then branch A would get 25% of the population.",
+          "type": "integer"
+        },
+        "features": {
+          "description": "An array of feature configurations.",
+          "items": {
+            "$ref": "#/$defs/ExperimentFeatureConfig"
+          },
+          "type": "array"
+        },
+        "firefoxLabsTitle": {
+          "description": "An optional string containing the title of the branch",
+          "type": "string"
+        },
+        "feature": {
+          "$ref": "#/$defs/DesktopPre95FeatureConfig",
+          "description": "The feature key must be provided with values to prevent crashes if the is encountered by Desktop clients earlier than version 95."
+        }
+      },
+      "required": [
+        "slug",
+        "ratio",
+        "features",
+        "feature"
+      ],
+      "type": "object"
+    },
+    "DesktopPre95FeatureConfig": {
+      "properties": {
+        "featureId": {
+          "const": "this-is-included-for-desktop-pre-95-support",
+          "type": "string"
+        },
+        "value": {
+          "type": "object"
+        },
+        "enabled": {
+          "const": false,
+          "type": "boolean"
+        }
+      },
+      "required": [
+        "featureId",
+        "value",
+        "enabled"
+      ],
+      "type": "object"
+    },
+    "ExperimentBucketConfig": {
+      "properties": {
+        "randomizationUnit": {
+          "$ref": "#/$defs/RandomizationUnit"
+        },
+        "namespace": {
+          "description": "Additional inputs to the hashing function.",
+          "type": "string"
+        },
+        "start": {
+          "description": "Index of the starting bucket of the range.",
+          "type": "integer"
+        },
+        "count": {
+          "description": "Number of buckets in the range.",
+          "type": "integer"
+        },
+        "total": {
+          "description": "The total number of buckets. You can assume this will always be 10000",
+          "type": "integer"
+        }
+      },
+      "required": [
+        "randomizationUnit",
+        "namespace",
+        "start",
+        "count",
+        "total"
+      ],
+      "type": "object"
+    },
+    "ExperimentFeatureConfig": {
+      "properties": {
+        "featureId": {
+          "description": "The identifier for the feature flag.",
+          "type": "string"
+        },
+        "value": {
+          "description": "The values that define the feature configuration. This should be validated against a schema.",
+          "type": "object"
+        }
+      },
+      "required": [
+        "featureId",
+        "value"
+      ],
+      "type": "object"
+    },
+    "ExperimentLocalizations": {
+      "additionalProperties": {
+        "additionalProperties": {
+          "type": "string"
+        },
+        "type": "object"
+      },
+      "description": "Per-locale localization substitutions. The top level key is the locale (e.g., \"en-US\" or \"fr\"). Each entry is a mapping of string IDs to their localized equivalents.",
+      "type": "object"
+    },
+    "ExperimentOutcome": {
+      "properties": {
+        "slug": {
+          "description": "Identifier for the outcome.",
+          "type": "string"
+        },
+        "priority": {
+          "description": "e.g., \"primary\" or \"secondary\".",
+          "type": "string"
+        }
+      },
+      "required": [
+        "slug",
+        "priority"
+      ],
+      "type": "object"
+    },
+    "RandomizationUnit": {
+      "description": "A unique, stable indentifier for the user used as an input to bucket hashing.",
+      "enum": [
+        "normandy_id",
+        "nimbus_id",
+        "user_id",
+        "group_id"
+      ],
+      "type": "string"
+    }
+  }
+}

schemas/schemas/DesktopNimbusExperiment.schema.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://json-schema.org/draft/2019-09/schema",
   "title": "DesktopNimbusExperiment",
-  "description": "A Nimbus experiment for Firefox Desktop.",
+  "description": "A Nimbus experiment for Firefox Desktop. This schema is less strict than DesktopAllVersionsNimbusExperiment and is intended for use in Firefox Desktop.",
   "type": "object",
   "properties": {
     "schemaVersion": {
@@ -270,10 +270,6 @@
           },
           "type": "array"
         },
-        "feature": {
-          "$ref": "#/$defs/DesktopTombstoneFeatureConfig",
-          "description": "The feature key must be provided with values to prevent crashes if the is encountered by Desktop clients earlier than version 95."
-        },
         "firefoxLabsTitle": {
           "description": "An optional string containing the title of the branch",
           "type": "string"
@@ -282,29 +278,7 @@
       "required": [
         "slug",
         "ratio",
-        "features",
+        "features"
-        "feature"
-      ],
-      "type": "object"
-    },
-    "DesktopTombstoneFeatureConfig": {
-      "properties": {
-        "featureId": {
-          "const": "unused-feature-id-for-legacy-support",
-          "type": "string"
-        },
-        "value": {
-          "type": "object"
-        },
-        "enabled": {
-          "const": false,
-          "type": "boolean"
-        }
-      },
-      "required": [
-        "featureId",
-        "value",
-        "enabled"
       ],
       "type": "object"
     },