Merge branch 'main' into shredder_mitigation_kpi_search

.pre-commit-config.yaml

@@ -6,11 +6,11 @@ repos:
   - id: no-commit-to-branch
     args: [--branch, main, --branch, master]
 - repo: https://github.com/psf/black
-  rev: 24.8.0
+  rev: 24.10.0
   hooks:
   - id: black
 - repo: https://github.com/PyCQA/flake8
-  rev: 7.0.0
+  rev: 4.0.1
   hooks:
   - id: flake8
     exclude: dags/
@@ -20,7 +20,7 @@ repos:
   - id: yamllint
     args: [-c, .yamllint.yaml, .]
 - repo: https://github.com/PyCQA/isort
-  rev: 5.13.2
+  rev: 5.12.0
   hooks:
   - id: isort
 # This pre-commit hook has been archived on Nov 2023
@@ -35,9 +35,8 @@ repos:
   hooks:
   - id: mypy
     exclude: sql/.+/.+/.+/query\.py$
-    additional_dependencies: [types-pytz==2024.2.0.20240913,
+    additional_dependencies: [types-pytz==2024.2.0.20241221,
                               types-ujson==5.10.0.20240515,
-                              types-python-dateutil==2.9.0.20240906,
-                              types-requests==2.32.0.20240914,
-                              types-attrs==19.1.0,
+                              types-python-dateutil==2.9.0.20241206,
+                              types-requests==2.32.0.20241016,
                               types-PyYAML==6.0.12.20240917]

bigquery_etl/cli/metadata.py

@@ -10,7 +10,11 @@ import click
 from dateutil.relativedelta import relativedelta
 from google.cloud import bigquery
 
-from bigquery_etl.metadata.parse_metadata import DatasetMetadata, Metadata
+from bigquery_etl.metadata.parse_metadata import (
+    DatasetMetadata,
+    Metadata,
+    WorkgroupAccessMetadata,
+)
 from bigquery_etl.metadata.publish_metadata import publish_metadata
 
 from ..cli.utils import (
@@ -57,6 +61,9 @@ def update(name: str, sql_dir: Optional[str], project_id: Optional[str]) -> None
     table_metadata_files = paths_matching_name_pattern(
         name, sql_dir, project_id=project_id, files=["metadata.yaml"]
     )
+    retained_dataset_roles = ConfigLoader.get(
+        "deprecation", "retain_dataset_roles", fallback=[]
+    )
 
     # create and populate the dataset metadata yaml file if it does not exist
     for table_metadata_file in table_metadata_files:
@@ -83,14 +90,41 @@ def update(name: str, sql_dir: Optional[str], project_id: Optional[str]) -> None
             # this overwrites existing workgroups
             table_metadata.workgroup_access = []
             table_metadata_updated = True
-            dataset_metadata.workgroup_access = []
+            dataset_metadata.workgroup_access = [
+                workgroup
+                for workgroup in dataset_metadata.workgroup_access
+                if workgroup.get("role") in retained_dataset_roles
+            ]
             dataset_metadata_updated = True
         else:
             if table_metadata.workgroup_access is None:
-                table_metadata.workgroup_access = (
-                    dataset_metadata.default_table_workgroup_access
-                )
-                table_metadata_updated = True
+                table_metadata.workgroup_access = []
+
+            for (
+                default_workgroup_access
+            ) in dataset_metadata.default_table_workgroup_access:
+                role_exists = False
+                for i, table_workgroup_access in enumerate(
+                    table_metadata.workgroup_access
+                ):
+                    if table_workgroup_access.role == default_workgroup_access.get(
+                        "role"
+                    ):
+                        role_exists = True
+                        table_metadata.workgroup_access[i].members = sorted(
+                            set(table_workgroup_access.members)
+                            | set(default_workgroup_access.get("members", []))
+                        )
+                        table_metadata_updated = True
+
+                if not role_exists:
+                    table_metadata.workgroup_access.append(
+                        WorkgroupAccessMetadata(
+                            role=default_workgroup_access["role"],
+                            members=default_workgroup_access.get("members", []),
+                        )
+                    )
+                    table_metadata_updated = True
 
         if dataset_metadata_updated:
             dataset_metadata.write(dataset_metadata_path)

bigquery_etl/cli/monitoring.py

@@ -790,8 +790,8 @@ def run(name, project_id, sql_dir, workspace, base_url, marker):
                 click.echo(
                     f"Trigger metric runs for {project}.{dataset}.{table}: {metric_ids}"
                 )
-                response = client.run_metric_batch(metric_ids=metric_ids)
-                for metric_info in response.metric_infos:
+                response = client.run_metric_batch_async(metric_ids=metric_ids)
+                for metric_info in response:
                     latest_metric_run = metric_info.latest_metric_runs[-1]
                     if (
                         latest_metric_run

bigquery_etl/dryrun.py

@@ -38,6 +38,21 @@ except ImportError:
     from backports.cached_property import cached_property  # type: ignore
 
 
+QUERY_PARAMETER_TYPE_VALUES = {
+    "DATE": "2019-01-01",
+    "DATETIME": "2019-01-01 00:00:00",
+    "TIMESTAMP": "2019-01-01 00:00:00",
+    "STRING": "foo",
+    "BOOL": True,
+    "FLOAT64": 1,
+    "FLOAT": 1,
+    "INT64": 1,
+    "INTEGER": 1,
+    "NUMERIC": 1,
+    "BIGNUMERIC": 1,
+}
+
+
 def get_credentials(auth_req: Optional[GoogleAuthRequest] = None):
     """Get GCP credentials."""
     auth_req = auth_req or GoogleAuthRequest()
@@ -204,28 +219,30 @@ class DryRun:
             sql = self.content
         else:
             sql = self.get_sql()
-        if self.metadata:
-            # use metadata to rewrite date-type query params as submission_date
-            date_params = [
-                query_param
-                for query_param in (
-                    self.metadata.scheduling.get("date_partition_parameter"),
-                    *(
-                        param.split(":", 1)[0]
-                        for param in self.metadata.scheduling.get("parameters", [])
-                        if re.fullmatch(r"[^:]+:DATE:{{.*ds.*}}", param)
-                    ),
+
+        query_parameters = []
+        scheduling_metadata = self.metadata.scheduling if self.metadata else {}
+        if date_partition_parameter := scheduling_metadata.get(
+            "date_partition_parameter", "submission_date"
+        ):
+            query_parameters.append(
+                bigquery.ScalarQueryParameter(
+                    date_partition_parameter,
+                    "DATE",
+                    QUERY_PARAMETER_TYPE_VALUES["DATE"],
                 )
-                if query_param and query_param != "submission_date"
-            ]
-            if date_params:
-                pattern = re.compile(
-                    "@("
-                    + "|".join(date_params)
-                    # match whole query parameter names
-                    + ")(?![a-zA-Z0-9_])"
+            )
+        for parameter in scheduling_metadata.get("parameters", []):
+            parameter_name, parameter_type, _ = parameter.strip().split(":", 2)
+            parameter_type = parameter_type.upper() or "STRING"
+            query_parameters.append(
+                bigquery.ScalarQueryParameter(
+                    parameter_name,
+                    parameter_type,
+                    QUERY_PARAMETER_TYPE_VALUES.get(parameter_type),
                 )
-                sql = pattern.sub("@submission_date", sql)
+            )
+
         project = basename(dirname(dirname(dirname(self.sqlfile))))
         dataset = basename(dirname(dirname(self.sqlfile)))
         try:
@@ -234,6 +251,10 @@ class DryRun:
                     "project": self.project or project,
                     "dataset": self.dataset or dataset,
                     "query": sql,
+                    "query_parameters": [
+                        query_parameter.to_api_repr()
+                        for query_parameter in query_parameters
+                    ],
                 }
 
                 if self.table:
@@ -257,11 +278,7 @@ class DryRun:
                     dry_run=True,
                     use_query_cache=False,
                     default_dataset=f"{project}.{dataset}",
-                    query_parameters=[
-                        bigquery.ScalarQueryParameter(
-                            "submission_date", "DATE", "2019-01-01"
-                        )
-                    ],
+                    query_parameters=query_parameters,
                 )
                 job = self.client.query(sql, job_config=job_config)
                 try:

bigquery_etl/metadata/validate_metadata.py

@@ -8,11 +8,12 @@ from pathlib import Path
 
 import click
 import yaml
+from google.cloud import bigquery
 
 from bigquery_etl.config import ConfigLoader
 from bigquery_etl.schema import SCHEMA_FILE, Schema
 
-from ..util import standard_args
+from ..util import extract_from_query_path, standard_args
 from ..util.common import extract_last_group_by_from_query, project_dirs
 from .parse_metadata import DatasetMetadata, Metadata
 
@@ -106,6 +107,7 @@ def validate_change_control(
 def validate_shredder_mitigation(query_dir, metadata):
     """Check queries with shredder mitigation label comply with requirements."""
     has_shredder_mitigation = SHREDDER_MITIGATION_LABEL in metadata.labels
+    table_not_empty = True
 
     if has_shredder_mitigation:
         schema_file = Path(query_dir) / SCHEMA_FILE
@@ -116,7 +118,44 @@ def validate_shredder_mitigation(query_dir, metadata):
         query_file = Path(query_dir) / "query.sql"
         query_group_by = extract_last_group_by_from_query(sql_path=query_file)
 
-        # Validate that the query group by is as required.
+        # Validate that this label is only applied to tables in version 1 if they're not empty
+        # If the table is empty, it should be backfilled before applying the label.
+        project, dataset, table = extract_from_query_path(Path(query_dir))
+        client = bigquery.Client(project=project)
+        error_message = (
+            f"The shredder-mitigation label can only be applied to existing and "
+            f"non-empty tables.\nEnsure that the table `{project}.{dataset}.{table}` is deployed"
+            f" and run a managed backfill without mitigation before applying this label to"
+            f" a new or empty table."
+            f"\n\nSubsequent backfills then can use the [shredder mitigation process]"
+            f"(https://mozilla.github.io/bigquery-etl/cookbooks/creating_a_derived_dataset/#initiating-the-backfill)."
+        )
+
+        # Check that the table exists and it's not empty.
+        query_table_is_not_empty = (
+            f"SELECT EXISTS (SELECT 1 "
+            f"FROM `{project}.{dataset}.INFORMATION_SCHEMA.TABLES` "
+            f"WHERE table_name = '{table}' LIMIT 1) AS not_empty;"
+        )
+
+        try:
+            table_not_empty = client.query(query_table_is_not_empty).result()
+        except Exception:
+            click.echo(
+                click.style(
+                    f"Table {project}.{dataset}.{table} not found or inaccessible."
+                    f" for validation. Please check that the name is correct and if the table"
+                    f" is in a private repository, ensure that it exists and has data before"
+                    f" running a backfill with shredder mitigation.",
+                    fg="yellow",
+                )
+            )
+
+        if table_not_empty is None or table_not_empty is False:
+            click.echo(click.style(error_message, fg="yellow"))
+            return False
+
+        # Validate that the query group by is explicit and as required.
         integers_in_group_by = False
         for e in query_group_by:
             try:
@@ -213,11 +252,11 @@ def validate_retention_policy_based_on_table_type(metadata, path):
         else None
     )
 
-    retention_exclusion_list = set(
-        ConfigLoader.get("retention_exclusion_list", fallback=[])
-    )
+    # retention_exclusion_list = set(
+    #     ConfigLoader.get("retention_exclusion_list", fallback=[])
+    # )
 
-    normalized_path = str(Path(path).parent)
+    # normalized_path = str(Path(path).parent)
     if expiration_days is not None and table_type == "aggregate":
         click.echo(
             click.style(
@@ -226,18 +265,19 @@ def validate_retention_policy_based_on_table_type(metadata, path):
             )
         )
         is_valid = False
-    if (
-        expiration_days is None
-        and table_type == "client_level"
-        and normalized_path not in retention_exclusion_list
-    ):
-        click.echo(
-            click.style(
-                f"ERROR: Table at {path} is an client level table and needs expiration_days to be set",
-                fg="red",
-            )
-        )
-        is_valid = False
+    # The below line of code should be uncommented when the retention project is completed
+    # if (
+    #     expiration_days is None
+    #     and table_type == "client_level"
+    #     and normalized_path not in retention_exclusion_list
+    # ):
+    #     click.echo(
+    #         click.style(
+    #             f"ERROR: Table at {path} is an client level table and needs expiration_days to be set",
+    #             fg="red",
+    #         )
+    #     )
+    #     is_valid = False
     return is_valid
 
 

bigquery_etl/query_scheduling/task.py

@@ -622,7 +622,7 @@ class Task:
         task.is_bigeye_check = True
         task.depends_on_past = False
         task.destination_table = None
-        task.retries = 0
+        task.retries = 1
         task.depends_on_fivetran = []
         task.referenced_tables = None
         task.depends_on = []

bigquery_etl/query_scheduling/templates/airflow_dag.j2

@@ -308,6 +308,7 @@ with DAG('{{ name }}', default_args=default_args{%+ if schedule_interval != None
             {#+ Avoid having concurrent tasks for ETLs that target the whole table. -#}
             task_concurrency=1,
             {%+ endif -%}
+            execution_timeout=datetime.timedelta(hours=1),
     {%+ else -%}
         {{ task.task_name }} = bigquery_etl_query(
             {% if name == "bqetl_default" -%}

bigquery_etl/shredder/config.py

@@ -19,6 +19,7 @@ from ..util.bigquery_id import qualified_table_id
 
 SHARED_PROD = "moz-fx-data-shared-prod"
 GLEAN_SCHEMA_ID = "glean_ping_1"
+GLEAN_MIN_SCHEMA_ID = "glean-min_ping_1"
 GLEAN_APP_LISTINGS_URL = "https://probeinfo.telemetry.mozilla.org/v2/glean/app-listings"
 
 
@@ -91,6 +92,7 @@ DeleteIndex = dict[DeleteTarget, DeleteSource | tuple[DeleteSource, ...]]
 
 CLIENT_ID = "client_id"
 GLEAN_CLIENT_ID = "client_info.client_id"
+GLEAN_USAGE_PROFILE_ID = "metrics.uuid.usage_profile_id"
 IMPRESSION_ID = "impression_id"
 USER_ID = "user_id"
 POCKET_ID = "pocket_id"
@@ -205,9 +207,6 @@ DELETE_TARGETS: DeleteIndex = {
     client_id_target(
         table="telemetry_derived.clients_last_seen_joined_v1"
     ): DESKTOP_SRC,
-    client_id_target(
-        table="telemetry_derived.clients_profile_per_install_affected_v1"
-    ): DESKTOP_SRC,
     client_id_target(table="telemetry_derived.core_clients_daily_v1"): DESKTOP_SRC,
     client_id_target(table="telemetry_derived.core_clients_last_seen_v1"): DESKTOP_SRC,
     client_id_target(table="telemetry_derived.event_events_v1"): DESKTOP_SRC,
@@ -298,8 +297,6 @@ DELETE_TARGETS: DeleteIndex = {
     DeleteTarget(table="telemetry_stable.sync_v4", field=SYNC_IDS): SYNC_SOURCES,
     DeleteTarget(table="telemetry_stable.sync_v5", field=SYNC_IDS): SYNC_SOURCES,
     # fxa
-    client_id_target(table="firefox_accounts_derived.events_daily_v1"): FXA_SRC,
-    client_id_target(table="firefox_accounts_derived.funnel_events_source_v1"): FXA_SRC,
     user_id_target(
         table="firefox_accounts_derived.fxa_amplitude_export_v1"
     ): FXA_HMAC_SRC,
@@ -533,20 +530,23 @@ def find_glean_targets(
     """
     datasets = {dataset.dataset_id for dataset in client.list_datasets(project)}
 
-    glean_stable_tables = [
-        table
-        for tables in pool.map(
-            client.list_tables,
-            [
-                bigquery.DatasetReference(project, dataset_id)
-                for dataset_id in datasets
-                if dataset_id.endswith("_stable")
-            ],
-            chunksize=1,
-        )
-        for table in tables
-        if table.labels.get("schema_id") == GLEAN_SCHEMA_ID
-    ]
+    def stable_tables_by_schema(schema_id):
+        return [
+            table
+            for tables in pool.map(
+                client.list_tables,
+                [
+                    bigquery.DatasetReference(project, dataset_id)
+                    for dataset_id in datasets
+                    if dataset_id.endswith("_stable")
+                ],
+                chunksize=1,
+            )
+            for table in tables
+            if table.labels.get("schema_id") == schema_id
+        ]
+
+    glean_stable_tables = stable_tables_by_schema(GLEAN_SCHEMA_ID)
 
     channel_to_app_name = get_glean_channel_to_app_name_mapping()
 
@@ -627,6 +627,16 @@ def find_glean_targets(
     manually_added_tables = {target.table for target in DELETE_TARGETS.keys()}
     skipped_tables = manually_added_tables | GLEAN_IGNORE_LIST
 
+    # Handle custom deletion requests for usage_reporting pings
+    glean_min_stable_tables = stable_tables_by_schema(GLEAN_MIN_SCHEMA_ID)
+    usage_reporting_sources: dict[str, tuple[DeleteSource, ...]] = defaultdict(tuple)
+    for table in glean_min_stable_tables:
+        if table.table_id == "usage_deletion_request_v1":
+            source = DeleteSource(
+                qualified_table_id(table), GLEAN_USAGE_PROFILE_ID, project
+            )
+            usage_reporting_sources[table.dataset_id] += (source,)
+
     return {
         **{
             # glean stable tables that have a source
@@ -675,6 +685,18 @@ def find_glean_targets(
             and not table.table_id.startswith(derived_source_prefix)
             and qualified_table_id(table) not in skipped_tables
         },
+        **{
+            # usage_reporting tables via custom usage_deletion_request ping
+            DeleteTarget(
+                table=qualified_table_id(table),
+                # field must be repeated for each deletion source
+                field=(GLEAN_USAGE_PROFILE_ID,)
+                * len(usage_reporting_sources[table.dataset_id]),
+            ): usage_reporting_sources[table.dataset_id]
+            for table in glean_min_stable_tables
+            if table.table_id == "usage_reporting_v1"
+            and table.dataset_id in usage_reporting_sources
+        },
     }
 
 

bigquery_etl/shredder/delete.py

@@ -14,6 +14,7 @@ from typing import Callable, Iterable, Optional, Tuple, Union
 
 from google.api_core.exceptions import NotFound
 from google.cloud import bigquery
+from google.cloud.bigquery import QueryJob
 
 from ..format_sql.formatter import reformat
 from ..util import standard_args
@@ -148,6 +149,16 @@ parser.add_argument(
 )
 
 
+@dataclass
+class DeleteJobResults:
+    """Subset of a bigquery job object retaining only fields that are needed."""
+
+    job_id: str
+    total_bytes_processed: Optional[int]
+    num_dml_affected_rows: Optional[int]
+    destination: str
+
+
 def record_state(client, task_id, job, dry_run, start_date, end_date, state_table):
     """Record the job for task_id in state_table."""
     if state_table is not None:
@@ -180,11 +191,11 @@ def wait_for_job(
     create_job,
     check_table_existence=False,
     **state_kwargs,
-):
+) -> DeleteJobResults:
     """Get a job from state or create a new job, and wait for the job to complete."""
     job = None
     if task_id in states:
-        job = client.get_job(**FULL_JOB_ID_RE.fullmatch(states[task_id]).groupdict())
+        job = client.get_job(**FULL_JOB_ID_RE.fullmatch(states[task_id]).groupdict())  # type: ignore[union-attr]
         if job.errors:
             logging.info(f"Previous attempt failed, retrying for {task_id}")
             job = None
@@ -209,7 +220,20 @@ def wait_for_job(
     if not dry_run and not job.ended:
         logging.info(f"Waiting on {full_job_id(job)} for {task_id}")
         job.result()
-    return job
+
+    try:
+        bytes_processed = job.total_bytes_processed
+    except AttributeError:
+        bytes_processed = 0
+
+    return DeleteJobResults(
+        job_id=job.job_id,
+        total_bytes_processed=bytes_processed,
+        num_dml_affected_rows=(
+            job.num_dml_affected_rows if isinstance(job, QueryJob) else None
+        ),
+        destination=job.destination,
+    )
 
 
 def get_task_id(target, partition_id):
@@ -419,13 +443,12 @@ def delete_from_partition_with_sampling(
             copy_job.total_bytes_processed = sum(
                 [r.total_bytes_processed for r in results]
             )
-            copy_job.num_dml_affected_rows = None
             return copy_job
         else:
             # copy job doesn't have dry runs so dry run base partition for byte estimate
             return client.query(
-                f"SELECT * FROM {target_table}",
-                job_config=bigquery.QueryJobConfig(dry_run=True, use_legacy_sql=True),
+                f"SELECT * FROM `{sql_table_id(target)}` WHERE {partition.condition}",
+                job_config=bigquery.QueryJobConfig(dry_run=True),
             )
 
     return partial(
@@ -789,21 +812,13 @@ def main():
         jobs_by_table[tasks[i].table].append(job)
     bytes_processed = rows_deleted = 0
     for table, jobs in jobs_by_table.items():
-        table_bytes_processed = sum(
-            job.total_bytes_processed or 0
-            for job in jobs
-            if isinstance(job, bigquery.QueryJob)
-        )
+        table_bytes_processed = sum(job.total_bytes_processed or 0 for job in jobs)
         bytes_processed += table_bytes_processed
         table_id = sql_table_id(table)
         if args.dry_run:
             logging.info(f"Would scan {table_bytes_processed} bytes from {table_id}")
         else:
-            table_rows_deleted = sum(
-                job.num_dml_affected_rows or 0
-                for job in jobs
-                if isinstance(job, bigquery.QueryJob)
-            )
+            table_rows_deleted = sum(job.num_dml_affected_rows or 0 for job in jobs)
             rows_deleted += table_rows_deleted
             logging.info(
                 f"Scanned {table_bytes_processed} bytes and "

bigquery_etl/util/common.py

@@ -56,7 +56,10 @@ def project_dirs(project_id=None, sql_dir=None) -> List[str]:
         sql_dir = ConfigLoader.get("default", "sql_dir", fallback="sql")
     if project_id is None:
         return [
-            os.path.join(sql_dir, project_dir) for project_dir in os.listdir(sql_dir)
+            os.path.join(sql_dir, project_dir)
+            for project_dir in os.listdir(sql_dir)
+            # sql/ can contain files like bigconfig.yml
+            if os.path.isdir(os.path.join(sql_dir, project_dir))
         ]
     else:
         return [os.path.join(sql_dir, project_id)]

bqetl_project.yaml

@@ -15,6 +15,7 @@ default:
   - moz-fx-data-pioneer-nonprod
   - moz-fx-data-pioneer-prod
   - moz-fx-data-glam-prod-fca7
+  - moz-fx-glam-prod
   public_project: mozilla-public-data
 
 monitoring:
@@ -42,6 +43,7 @@ dry_run:
   - sql/moz-fx-data-shared-prod/accounts_backend_external/nonprod_emails_v1/query.sql
   - sql/moz-fx-data-shared-prod/accounts_backend_external/accounts_v1/query.sql
   - sql/moz-fx-data-shared-prod/accounts_backend_external/emails_v1/query.sql
+  - sql/moz-fx-data-shared-prod/monitoring_derived/table_storage_v1/query.sql
   # uses time travel, will error on dates prior to the time travel window
   - sql/moz-fx-data-shared-prod/accounts_backend_derived/monitoring_db_counts_v1/query.sql
   - sql/moz-fx-data-shared-prod/accounts_db_external/**/*.sql
@@ -199,6 +201,7 @@ dry_run:
   # Dataset sql/glam-fenix-dev:glam_etl was not found
   - sql/glam-fenix-dev/glam_etl/**/*.sql
   - sql/moz-fx-data-glam-prod-fca7/glam_etl/**/*.sql
+  - sql/moz-fx-glam-prod/glam_etl/**/*.sql
   # Source table does not have a verifiable schema
   - sql/moz-fx-data-shared-prod/monitoring_derived/outerbounds_cost_per_flow_run_v1/view.sql
   - sql/moz-fx-data-shared-prod/monitoring_derived/outerbounds_cost_per_flow_v1/view.sql
@@ -214,6 +217,9 @@ dry_run:
   # Tests
   - sql/moz-fx-data-test-project/test/simple_view/view.sql
 
+deprecation:
+  retain_dataset_roles: # the following roles are retain permissions on datasets when a dataset table is deprecated
+  - roles/bigquery.dataEditor  # operational access
 
 format:
   skip:
@@ -337,6 +343,49 @@ generate:
       events_tables: # overwrite event tables
         pine: # Probe info returns pings that don't have tables, only use events_v1
         - events_v1
+    bigconfig:
+      skip_apps:
+      - firefox_echo_show
+      - firefox_fire_tv
+      - thunderbird_android
+      - org_mozilla_social_nightly
+      - mozilla_vpn
+      - mozillavpn_backend_cirrus
+      - accounts_backend
+      - accounts_cirrus
+      - burnham
+      - firefox_crashreporter
+      - firefox_reality
+      - firefox_reality_pc
+      - lockwise_android
+      - mach
+      - monitor_backend
+      - monitor_cirrus
+      - moso_mastodon_backend
+      - mozphab
+      - mozregression
+      skip_app_metrics:
+      - thunderbird_desktop
+    bqetl_checks:
+      skip_apps:
+      - mozilla_vpn
+      - mozillavpn_backend_cirrus
+      - accounts_backend
+      - accounts_cirrus
+      - burnham
+      - firefox_crashreporter
+      - firefox_reality
+      - firefox_reality_pc
+      - lockwise_android
+      - mach
+      - monitor_backend
+      - monitor_cirrus
+      - moso_mastodon_backend
+      - mozphab
+      - mozregression
+  stable_views:
+    skip_datasets:
+    - mlhackweek_search
 
 retention_exclusion_list:
 - sql/moz-fx-data-shared-prod/search_derived/acer_cohort_v1

dags.yaml

@@ -431,7 +431,7 @@ bqetl_monitoring:
     used for monitoring of the data platform.
   default_args:
     owner: ascholtz@mozilla.com
-    email: ["ascholtz@mozilla.com"]
+    email: ["ascholtz@mozilla.com", "kwindau@mozilla.com"]
     start_date: "2018-10-30"
     retries: 2
     retry_delay: 30m
@@ -1490,7 +1490,7 @@ bqetl_glam_refresh_aggregates:
     - impact/tier_2
     - repo/bigquery-etl
 
-bqetl_glam_refresh_aggregates_fog_release:
+bqetl_glam_refresh_aggregates_release:
   default_args:
     depends_on_past: false
     email:
@@ -1501,7 +1501,7 @@ bqetl_glam_refresh_aggregates_fog_release:
     retries: 2
     retry_delay: 30m
     start_date: '2024-12-10'
-  description: Update GLAM FOG release tables that serve aggregated data.
+  description: Update GLAM FOG and Fenix release tables that serve aggregated data.
   repo: bigquery-etl
   schedule_interval: 0 18 * * 6
   tags:
@@ -2037,3 +2037,60 @@ bqetl_event_aggregates:
     - impact/tier_3
   repo: bigquery-etl
   schedule_interval: 40 16 * * *
+
+bqetl_fx_cert_error_privacy_dashboard:
+  description: |
+    This DAG builds the desktop event_aggregates table for unique users count
+  default_args:
+    depends_on_past: false
+    owner: akommasani@mozilla.com
+    email:
+      - telemetry-alerts@mozilla.com
+      - akommasani@mozilla.com
+    email_on_failure: true
+    email_on_retry: false
+    start_date: "2024-12-19"
+    retries: 2
+    retry_delay: 5m
+  tags:
+    - impact/tier_3
+  repo: bigquery-etl
+  schedule_interval: 40 16 * * *
+
+bqetl_default_browser_aggregates:
+  default_args:
+    depends_on_past: false
+    email:
+      - telemetry-alerts@mozilla.com
+      - wichan@mozilla.com
+    email_on_failure: true
+    email_on_retry: false
+    owner: wichan@mozilla.com
+    retries: 2
+    retry_delay: 30m
+    start_date: '2024-12-20'
+  description: |
+    This DAG builds daily aggregate tables for default browser tables
+  repo: bigquery-etl
+  schedule_interval: 0 22 * * *
+  tags:
+    - impact/tier_3
+
+bqetl_dsktp_acqstn_fnnl:
+  description: |
+    This DAG builds the Desktop Acquisiton Funnel aggregate table
+  default_args:
+    depends_on_past: false
+    owner: kwindau@mozilla.com
+    email:
+      - telemetry-alerts@mozilla.com
+      - kwindau@mozilla.com
+    email_on_failure: true
+    email_on_retry: false
+    start_date: "2025-01-08"
+    retries: 2
+    retry_delay: 5m
+  tags:
+    - impact/tier_2
+  repo: bigquery-etl
+  schedule_interval: 30 11 * * *

docs/cookbooks/common_workflows.md

@@ -126,7 +126,7 @@ The following is an example to update a new field in `telemetry_derived.clients_
 
 ## Remove a field from a table schema
 
-Deleting a field from an existing table schema should be done only when is totally neccessary. If you decide to delete it:
+Deleting a field from an existing table schema should be done only when is totally necessary. If you decide to delete it:
 1. Validate if there is data in the column and make sure data it is either backed up or it can be reprocessed.
 1. Follow [Big Query docs](https://cloud.google.com/bigquery/docs/managing-table-schemas#deleting_columns_from_a_tables_schema_definition) recommendations for deleting.
 1. If the column size exceeds the allowed limit, consider setting the field as NULL. See this [search_clients_daily_v8](https://github.com/mozilla/bigquery-etl/pull/2463) PR for an example.

docs/cookbooks/creating_a_derived_dataset.md

@@ -268,11 +268,13 @@ For our example:
   bqetl backfill create <project>.<dataset>.<table> --start_date=<YYYY-MM-DD> --end_date=<YYYY-MM-DD>
   ```
 
-  - If the backfill requires [shredder_mitigation](https://docs.telemetry.mozilla.org/cookbooks/data_modeling/shredder_mitigation) to maintain metrics stable, use the `--shredder_mitigation` parameter in the backfill command:
-
-  ```bash
-  bqetl backfill create <project>.<dataset>.<table> --start_date=<YYYY-MM-DD> --end_date=<YYYY-MM-DD> --shredder_mitigation
-  ```
+  - If the table's metadata has the label `shredder_mitigation: true`, use the process to run a [backfill with shredder_mitigation](https://docs.telemetry.mozilla.org/cookbooks/data_modeling/shredder_mitigation#running-a-managed-backfill-with-shredder-mitigation):
+    - Bump the version of the query.
+    - Make the necessary updates to the new version of the query and schema.
+    - Create the managed backfill for the new version of the query, including the parameter `--shredder_mitigation`.
+      ```bash
+      bqetl backfill create <project>.<dataset>.<table> --start_date=<YYYY-MM-DD> --end_date=<YYYY-MM-DD> --shredder_mitigation
+      ```
 
 2. Fill out the missing details:
   - Watchers: Mozilla Emails for users that should be notified via Slack about backfill progress.

docs/reference/configuration.md

@@ -16,7 +16,6 @@ dry_run:
 views:
   skip_validation:
   - sql/moz-fx-data-test-project/test/simple_view/view.sql
-  - sql/moz-fx-data-shared-prod/mlhackweek_search/events/view.sql
   - sql/moz-fx-data-shared-prod/**/client_deduplication/view.sql
   # - ...
   skip_publishing:

pyproject.toml

@@ -11,7 +11,7 @@ description = "Tooling for building derived datasets in BigQuery"
 readme = "README.md"
 requires-python = ">=3.11"
 dynamic = ["dependencies"]
-version = "2024.5.1"
+version = "2025.1.1"
 
 [project.urls]
 Homepage = "https://github.com/mozilla/bigquery-etl"

requirements.in

@@ -1,34 +1,35 @@
 attrs==24.3.0
-authlib==1.3.2
-bigeye-sdk==0.4.94
+authlib==1.4.0
+bigeye-sdk==0.4.97
 black==24.10.0
 cattrs==24.1.2
-click==8.1.7
+click==8.1.8
+cryptography==44.0.0
 exceptiongroup==1.2.2 # for backwards compatibility with python < 3.11
 flake8<5 # pytest-flake8 does not support flake8 5+
-gcsfs==2024.10.0
+gcsfs==2024.12.0
 gcloud==0.18.3
-gitpython==3.1.43
+gitpython==3.1.44
 google-auth>=2.30.0  # To try to fix "Compute Engine Metadata server call to universe/universe_domain returned 404" errors.
-google-cloud-bigquery==3.27.0
+google-cloud-bigquery==3.29.0
 google-cloud-bigquery-storage[fastavro]==2.27.0
 google-cloud-datacatalog-lineage==0.3.11
 google-cloud-storage==2.19.0
-Jinja2==3.1.4
+Jinja2==3.1.5
 jsonschema==4.23.0
 PyJWT==2.10.1
 markdown-include==0.8.1
 mdx_truly_sane_lists==1.3
 mkdocs==1.6.1
-mkdocs-material==9.5.49
+mkdocs-material==9.5.50
 mkdocs-awesome-pages-plugin==2.9.3
 mozilla-metric-config-parser==2024.11.1
 mozilla-schema-generator==0.5.1
 pandas==2.2.3
 pathos==0.3.3
 pip-tools==7.4.1
-pre-commit==4.0.1
-pyarrow==18.1.0
+pre-commit==4.1.0
+pyarrow==19.0.0
 pytest-black==0.6.0
 pytest-flake8==1.3.0
 pytest-isort==4.0.0
@@ -42,16 +43,16 @@ smart_open==6.4.0
 sqlglot==25.28.0
 sqlparse==0.5.3
 stripe==6.4.0
-symbolic==12.12.4
+symbolic==12.13.3
 siggen==2.2.20241029
 tomli==2.2.1 # for backwards compatibility with python < 3.11
 types-python-dateutil==2.9.0.20241206
-types-pytz==2024.2.0.20241003
-types-PyYAML==6.0.12.20240917
+types-pytz==2024.2.0.20241221
+types-PyYAML==6.0.12.20241230
 types-requests==2.32.0.20241016
 types-ujson==5.10.0.20240515
 typing==3.7.4.3
 ujson==5.10.0
 yamllint==1.35.1
 requests==2.32.3
-responses==0.25.3
+responses==0.25.6

requirements.txt

@@ -121,9 +121,9 @@ attrs==24.3.0 \
     #   mozilla-metric-config-parser
     #   pytest-mypy
     #   referencing
-authlib==1.3.2 \
-    --hash=sha256:4b16130117f9eb82aa6eec97f6dd4673c3f960ac0283ccdae2897ee4bc030ba2 \
-    --hash=sha256:ede026a95e9f5cdc2d4364a52103f5405e75aa156357e831ef2bfd0bc5094dfc
+authlib==1.4.0 \
+    --hash=sha256:1c1e6608b5ed3624aeeee136ca7f8c120d6f51f731aa152b153d54741840e1f2 \
+    --hash=sha256:4bb20b978c8b636222b549317c1815e1fe62234fc1c5efe8855d84aebf3a74e3
     # via -r requirements.in
 babel==2.12.1 \
     --hash=sha256:b4246fb7677d3b98f501a39d43396d3cafdc8eadb045f4a31be01863f655c610 \
@@ -132,9 +132,9 @@ babel==2.12.1 \
 betterproto[compiler]==1.2.5 \
     --hash=sha256:74a3ab34646054f674d236d1229ba8182dc2eae86feb249b8590ef496ce9803d
     # via bigeye-sdk
-bigeye-sdk==0.4.94 \
-    --hash=sha256:d17e50629ecf3273bc7ca8e4ff9883d8fb59e1111776a02ef9cdc07399e1306c \
-    --hash=sha256:ff230eb0fcfdfd72932ec83975ede59c7f1c2d2fed89286fb70463f3752fcef7
+bigeye-sdk==0.4.97 \
+    --hash=sha256:90d44f8cfc3170e984f39a684f24e38b7fa2268172b82a6bf601c96a5c1dfe48 \
+    --hash=sha256:fd2a0db64486a3c2a3dfc49ae66c8604d2d7a4b5ac8be22f4779edaf49417af1
     # via -r requirements.in
 black==24.10.0 \
     --hash=sha256:14b3502784f09ce2443830e3133dacf2c0110d45191ed470ecb04d0f5f6fcb0f \
@@ -278,9 +278,9 @@ charset-normalizer==2.1.1 \
     --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 \
     --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f
     # via requests
-click==8.1.7 \
-    --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \
-    --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de
+click==8.1.8 \
+    --hash=sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2 \
+    --hash=sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a
     # via
     #   -r requirements.in
     #   black
@@ -298,40 +298,36 @@ commonmark==0.9.1 \
     --hash=sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60 \
     --hash=sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9
     # via rich
-cryptography==42.0.4 \
-    --hash=sha256:01911714117642a3f1792c7f376db572aadadbafcd8d75bb527166009c9f1d1b \
-    --hash=sha256:0e89f7b84f421c56e7ff69f11c441ebda73b8a8e6488d322ef71746224c20fce \
-    --hash=sha256:12d341bd42cdb7d4937b0cabbdf2a94f949413ac4504904d0cdbdce4a22cbf88 \
-    --hash=sha256:15a1fb843c48b4a604663fa30af60818cd28f895572386e5f9b8a665874c26e7 \
-    --hash=sha256:1cdcdbd117681c88d717437ada72bdd5be9de117f96e3f4d50dab3f59fd9ab20 \
-    --hash=sha256:1df6fcbf60560d2113b5ed90f072dc0b108d64750d4cbd46a21ec882c7aefce9 \
-    --hash=sha256:3c6048f217533d89f2f8f4f0fe3044bf0b2090453b7b73d0b77db47b80af8dff \
-    --hash=sha256:3e970a2119507d0b104f0a8e281521ad28fc26f2820687b3436b8c9a5fcf20d1 \
-    --hash=sha256:44a64043f743485925d3bcac548d05df0f9bb445c5fcca6681889c7c3ab12764 \
-    --hash=sha256:4e36685cb634af55e0677d435d425043967ac2f3790ec652b2b88ad03b85c27b \
-    --hash=sha256:5f8907fcf57392cd917892ae83708761c6ff3c37a8e835d7246ff0ad251d9298 \
-    --hash=sha256:69b22ab6506a3fe483d67d1ed878e1602bdd5912a134e6202c1ec672233241c1 \
-    --hash=sha256:6bfadd884e7280df24d26f2186e4e07556a05d37393b0f220a840b083dc6a824 \
-    --hash=sha256:6d0fbe73728c44ca3a241eff9aefe6496ab2656d6e7a4ea2459865f2e8613257 \
-    --hash=sha256:6ffb03d419edcab93b4b19c22ee80c007fb2d708429cecebf1dd3258956a563a \
-    --hash=sha256:810bcf151caefc03e51a3d61e53335cd5c7316c0a105cc695f0959f2c638b129 \
-    --hash=sha256:831a4b37accef30cccd34fcb916a5d7b5be3cbbe27268a02832c3e450aea39cb \
-    --hash=sha256:887623fe0d70f48ab3f5e4dbf234986b1329a64c066d719432d0698522749929 \
-    --hash=sha256:a0298bdc6e98ca21382afe914c642620370ce0470a01e1bef6dd9b5354c36854 \
-    --hash=sha256:a1327f280c824ff7885bdeef8578f74690e9079267c1c8bd7dc5cc5aa065ae52 \
-    --hash=sha256:c1f25b252d2c87088abc8bbc4f1ecbf7c919e05508a7e8628e6875c40bc70923 \
-    --hash=sha256:c3a5cbc620e1e17009f30dd34cb0d85c987afd21c41a74352d1719be33380885 \
-    --hash=sha256:ce8613beaffc7c14f091497346ef117c1798c202b01153a8cc7b8e2ebaaf41c0 \
-    --hash=sha256:d2a27aca5597c8a71abbe10209184e1a8e91c1fd470b5070a2ea60cafec35bcd \
-    --hash=sha256:dad9c385ba8ee025bb0d856714f71d7840020fe176ae0229de618f14dae7a6e2 \
-    --hash=sha256:db4b65b02f59035037fde0998974d84244a64c3265bdef32a827ab9b63d61b18 \
-    --hash=sha256:e09469a2cec88fb7b078e16d4adec594414397e8879a4341c6ace96013463d5b \
-    --hash=sha256:e53dc41cda40b248ebc40b83b31516487f7db95ab8ceac1f042626bc43a2f992 \
-    --hash=sha256:f1e85a178384bf19e36779d91ff35c7617c885da487d689b05c1366f9933ad74 \
-    --hash=sha256:f47be41843200f7faec0683ad751e5ef11b9a56a220d57f300376cd8aba81660 \
-    --hash=sha256:fb0cef872d8193e487fc6bdb08559c3aa41b659a7d9be48b2e10747f47863925 \
-    --hash=sha256:ffc73996c4fca3d2b6c1c8c12bfd3ad00def8621da24f547626bf06441400449
+cryptography==44.0.0 \
+    --hash=sha256:1923cb251c04be85eec9fda837661c67c1049063305d6be5721643c22dd4e2b7 \
+    --hash=sha256:37d76e6863da3774cd9db5b409a9ecfd2c71c981c38788d3fcfaf177f447b731 \
+    --hash=sha256:3c672a53c0fb4725a29c303be906d3c1fa99c32f58abe008a82705f9ee96f40b \
+    --hash=sha256:404fdc66ee5f83a1388be54300ae978b2efd538018de18556dde92575e05defc \
+    --hash=sha256:4ac4c9f37eba52cb6fbeaf5b59c152ea976726b865bd4cf87883a7e7006cc543 \
+    --hash=sha256:62901fb618f74d7d81bf408c8719e9ec14d863086efe4185afd07c352aee1d2c \
+    --hash=sha256:660cb7312a08bc38be15b696462fa7cc7cd85c3ed9c576e81f4dc4d8b2b31591 \
+    --hash=sha256:708ee5f1bafe76d041b53a4f95eb28cdeb8d18da17e597d46d7833ee59b97ede \
+    --hash=sha256:761817a3377ef15ac23cd7834715081791d4ec77f9297ee694ca1ee9c2c7e5eb \
+    --hash=sha256:831c3c4d0774e488fdc83a1923b49b9957d33287de923d58ebd3cec47a0ae43f \
+    --hash=sha256:84111ad4ff3f6253820e6d3e58be2cc2a00adb29335d4cacb5ab4d4d34f2a123 \
+    --hash=sha256:8b3e6eae66cf54701ee7d9c83c30ac0a1e3fa17be486033000f2a73a12ab507c \
+    --hash=sha256:9e6fc8a08e116fb7c7dd1f040074c9d7b51d74a8ea40d4df2fc7aa08b76b9e6c \
+    --hash=sha256:a01956ddfa0a6790d594f5b34fc1bfa6098aca434696a03cfdbe469b8ed79285 \
+    --hash=sha256:abc998e0c0eee3c8a1904221d3f67dcfa76422b23620173e28c11d3e626c21bd \
+    --hash=sha256:b15492a11f9e1b62ba9d73c210e2416724633167de94607ec6069ef724fad092 \
+    --hash=sha256:be4ce505894d15d5c5037167ffb7f0ae90b7be6f2a98f9a5c3442395501c32fa \
+    --hash=sha256:c5eb858beed7835e5ad1faba59e865109f3e52b3783b9ac21e7e47dc5554e289 \
+    --hash=sha256:cd4e834f340b4293430701e772ec543b0fbe6c2dea510a5286fe0acabe153a02 \
+    --hash=sha256:d2436114e46b36d00f8b72ff57e598978b37399d2786fd39793c36c6d5cb1c64 \
+    --hash=sha256:eb33480f1bad5b78233b0ad3e1b0be21e8ef1da745d8d2aecbb20671658b9053 \
+    --hash=sha256:eca27345e1214d1b9f9490d200f9db5a874479be914199194e746c893788d417 \
+    --hash=sha256:ed3534eb1090483c96178fcb0f8893719d96d5274dfde98aa6add34614e97c8e \
+    --hash=sha256:f3f6fdfa89ee2d9d496e2c087cebef9d4fcbb0ad63c40e821b39f74bf48d9c5e \
+    --hash=sha256:f53c2c87e0fb4b0c00fa9571082a057e37690a8f12233306161c8f4b819960b7 \
+    --hash=sha256:f5e7cb1e5e56ca0933b4873c0220a78b773b24d40d186b6738080b73d3d0a756 \
+    --hash=sha256:f677e1268c4e23420c3acade68fac427fffcb8d19d7df95ed7ad17cdef8404f4
     # via
+    #   -r requirements.in
     #   authlib
     #   secretstorage
 decorator==5.1.1 \
@@ -348,9 +344,9 @@ dill==0.3.9 \
     # via
     #   multiprocess
     #   pathos
-distlib==0.3.6 \
-    --hash=sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46 \
-    --hash=sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e
+distlib==0.3.9 \
+    --hash=sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87 \
+    --hash=sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403
     # via virtualenv
 exceptiongroup==1.2.2 \
     --hash=sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b \
@@ -402,9 +398,9 @@ fastavro==1.9.0 \
     --hash=sha256:f803c33f4fd4e3bfc17bbdbf3c036fbcb92a1f8e6bd19a035800518479ce6b36 \
     --hash=sha256:fb7e3a058a169d2c8bd19dfcbc7ae14c879750ce49fbaf3c436af683991f7eae
     # via google-cloud-bigquery-storage
-filelock==3.9.0 \
-    --hash=sha256:7b319f24340b51f55a2bf7a12ac0755a9b03e718311dac567a0f4f7fabd2f5de \
-    --hash=sha256:f58d535af89bb9ad5cd4df046f741f8553a418c01a7856bf0d173bbc9f6bd16d
+filelock==3.16.1 \
+    --hash=sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0 \
+    --hash=sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435
     # via
     #   pytest-mypy
     #   virtualenv
@@ -492,16 +488,16 @@ frozenlist==1.3.3 \
     # via
     #   aiohttp
     #   aiosignal
-fsspec==2024.10.0 \
-    --hash=sha256:03b9a6785766a4de40368b88906366755e2819e758b83705c88cd7cb5fe81871 \
-    --hash=sha256:eda2d8a4116d4f2429db8550f2457da57279247dd930bb12f821b58391359493
+fsspec==2024.12.0 \
+    --hash=sha256:670700c977ed2fb51e0d9f9253177ed20cbde4a3e5c0283cc5385b5870c8533f \
+    --hash=sha256:b520aed47ad9804237ff878b504267a3b0b441e97508bd6d2d8774e3db85cee2
     # via gcsfs
 gcloud==0.18.3 \
     --hash=sha256:0af2dec59fce20561752f86e42d981c6a255e306a6c5e5d1fa3d358a8857e4fb
     # via -r requirements.in
-gcsfs==2024.10.0 \
-    --hash=sha256:5df54cfe568e8fdeea5aafa7fed695cdc69a9a674e991ca8c1ce634f5df1d314 \
-    --hash=sha256:bb2d23547e61203ea2dda5fa6c4b91a0c34b74ebe8bb6ab1926f6c33381bceb2
+gcsfs==2024.12.0 \
+    --hash=sha256:e672413922108300ebc1fe78b8f99f3c7c1b94e7e088f5a6dc88de6d5a93d156 \
+    --hash=sha256:ec88e48f77e466723705458af85dda238e43aa69fac071efd98829d06e9f095a
     # via -r requirements.in
 ghp-import==2.1.0 \
     --hash=sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619 \
@@ -511,9 +507,9 @@ gitdb==4.0.10 \
     --hash=sha256:6eb990b69df4e15bad899ea868dc46572c3f75339735663b81de79b06f17eb9a \
     --hash=sha256:c286cf298426064079ed96a9e4a9d39e7f3e9bf15ba60701e95f5492f28415c7
     # via gitpython
-gitpython==3.1.43 \
-    --hash=sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c \
-    --hash=sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff
+gitpython==3.1.44 \
+    --hash=sha256:9e0e10cda9bed1ee64bc9a6de50e7e38a9c9943241cd7f585f6df3ed28011110 \
+    --hash=sha256:c87e30b26253bf5418b01b0660f818967f3c503193838337fe5e573331249269
     # via
     #   -r requirements.in
     #   mozilla-metric-config-parser
@@ -531,9 +527,9 @@ google-api-core[grpc]==2.17.1 \
     #   google-cloud-core
     #   google-cloud-datacatalog-lineage
     #   google-cloud-storage
-google-auth==2.37.0 \
-    --hash=sha256:0054623abf1f9c83492c63d3f47e77f0a544caa3d40b2d98e099a611c2dd5d00 \
-    --hash=sha256:42664f18290a6be591be5329a96fe30184be1a1badb7292a7f686a9659de9ca0
+google-auth==2.38.0 \
+    --hash=sha256:8285113607d3b80a3f1543b75962447ba8a09fe85783432a784fdeef6ac094c4 \
+    --hash=sha256:e7dae6694313f434a2727bf2906f27ad259bae090d7aa896590d86feec3d9d4a
     # via
     #   -r requirements.in
     #   gcsfs
@@ -548,9 +544,9 @@ google-auth-oauthlib==0.8.0 \
     --hash=sha256:40cc612a13c3336d5433e94e2adb42a0c88f6feb6c55769e44500fc70043a576 \
     --hash=sha256:81056a310fb1c4a3e5a7e1a443e1eb96593c6bbc55b26c0261e4d3295d3e6593
     # via gcsfs
-google-cloud-bigquery==3.27.0 \
-    --hash=sha256:379c524054d7b090fa56d0c22662cc6e6458a6229b6754c0e7177e3a73421d2c \
-    --hash=sha256:b53b0431e5ba362976a4cd8acce72194b4116cdf8115030c7b339b884603fcc3
+google-cloud-bigquery==3.29.0 \
+    --hash=sha256:5453a4eabe50118254eda9778f3d7dad413490de5f7046b5e66c98f5a1580308 \
+    --hash=sha256:fafc2b455ffce3bcc6ce0e884184ef50b6a11350a83b91e327fadda4d5566e72
     # via -r requirements.in
 google-cloud-bigquery-storage[fastavro]==2.27.0 \
     --hash=sha256:3bfa8f74a61ceaffd3bfe90be5bbef440ad81c1c19ac9075188cccab34bffc2b \
@@ -767,9 +763,9 @@ jeepney==0.8.0 \
     # via
     #   keyring
     #   secretstorage
-jinja2==3.1.4 \
-    --hash=sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369 \
-    --hash=sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d
+jinja2==3.1.5 \
+    --hash=sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb \
+    --hash=sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb
     # via
     #   -r requirements.in
     #   betterproto
@@ -934,9 +930,9 @@ mkdocs-get-deps==0.2.0 \
     --hash=sha256:162b3d129c7fad9b19abfdcb9c1458a651628e4b1dea628ac68790fb3061c60c \
     --hash=sha256:2bf11d0b133e77a0dd036abeeb06dec8775e46efa526dc70667d8863eefc6134
     # via mkdocs
-mkdocs-material==9.5.49 \
-    --hash=sha256:3671bb282b4f53a1c72e08adbe04d2481a98f85fed392530051f80ff94a9621d \
-    --hash=sha256:c3c2d8176b18198435d3a3e119011922f3e11424074645c24019c2dcf08a360e
+mkdocs-material==9.5.50 \
+    --hash=sha256:ae5fe16f3d7c9ccd05bb6916a7da7420cf99a9ce5e33debd9d40403a090d5825 \
+    --hash=sha256:f24100f234741f4d423a9d672a909d859668a4f404796be3cf035f10d6050385
     # via -r requirements.in
 mkdocs-material-extensions==1.3 \
     --hash=sha256:0297cc48ba68a9fdd1ef3780a3b41b534b0d0df1d1181a44676fda5f464eeadc \
@@ -1209,9 +1205,9 @@ pip-tools==7.4.1 \
     --hash=sha256:4c690e5fbae2f21e87843e89c26191f0d9454f362d8acdbd695716493ec8b3a9 \
     --hash=sha256:864826f5073864450e24dbeeb85ce3920cdfb09848a3d69ebf537b521f14bcc9
     # via -r requirements.in
-platformdirs==2.6.2 \
-    --hash=sha256:83c8f6d04389165de7c9b6f0c682439697887bca0aa2f1c87ef1826be3584490 \
-    --hash=sha256:e1fea1fe471b9ff8332e229df3cb7de4f53eeea4998d3b6bfff542115e998bd2
+platformdirs==4.3.6 \
+    --hash=sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907 \
+    --hash=sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb
     # via
     #   black
     #   mkdocs-get-deps
@@ -1232,9 +1228,9 @@ ppft==1.7.6.9 \
     --hash=sha256:73161c67474ea9d81d04bcdad166d399cff3f084d5d2dc21ebdd46c075bbc265 \
     --hash=sha256:dab36548db5ca3055067fbe6b1a17db5fee29f3c366c579a9a27cebb52ed96f0
     # via pathos
-pre-commit==4.0.1 \
-    --hash=sha256:80905ac375958c0444c65e9cebebd948b3cdb518f335a091a670a89d652139d2 \
-    --hash=sha256:efde913840816312445dc98787724647c65473daefe420785f885e8ed9a06878
+pre-commit==4.1.0 \
+    --hash=sha256:ae3f018575a588e30dfddfab9a05448bfbd6b73d78709617b5a2b853549716d4 \
+    --hash=sha256:d29e7cb346295bcc1cc75fc3e92e343495e3ea0196c9ec6ba53f49f10ab6ae7b
     # via -r requirements.in
 propcache==0.2.0 \
     --hash=sha256:00181262b17e517df2cd85656fcd6b4e70946fe62cd625b9d74ac9977b64d8d9 \
@@ -1366,49 +1362,49 @@ protobuf==4.21.12 \
     #   googleapis-common-protos
     #   grpcio-status
     #   proto-plus
-pyarrow==18.1.0 \
-    --hash=sha256:01c034b576ce0eef554f7c3d8c341714954be9b3f5d5bc7117006b85fcf302fe \
-    --hash=sha256:05a5636ec3eb5cc2a36c6edb534a38ef57b2ab127292a716d00eabb887835f1e \
-    --hash=sha256:0743e503c55be0fdb5c08e7d44853da27f19dc854531c0570f9f394ec9671d54 \
-    --hash=sha256:0ad4892617e1a6c7a551cfc827e072a633eaff758fa09f21c4ee548c30bcaf99 \
-    --hash=sha256:0b331e477e40f07238adc7ba7469c36b908f07c89b95dd4bd3a0ec84a3d1e21e \
-    --hash=sha256:11b676cd410cf162d3f6a70b43fb9e1e40affbc542a1e9ed3681895f2962d3d9 \
-    --hash=sha256:25dbacab8c5952df0ca6ca0af28f50d45bd31c1ff6fcf79e2d120b4a65ee7181 \
-    --hash=sha256:2c4dd0c9010a25ba03e198fe743b1cc03cd33c08190afff371749c52ccbbaf76 \
-    --hash=sha256:36ac22d7782554754a3b50201b607d553a8d71b78cdf03b33c1125be4b52397c \
-    --hash=sha256:3b2e2239339c538f3464308fd345113f886ad031ef8266c6f004d49769bb074c \
-    --hash=sha256:3c35813c11a059056a22a3bef520461310f2f7eea5c8a11ef9de7062a23f8d56 \
-    --hash=sha256:4a4813cb8ecf1809871fd2d64a8eff740a1bd3691bbe55f01a3cf6c5ec869754 \
-    --hash=sha256:4f443122c8e31f4c9199cb23dca29ab9427cef990f283f80fe15b8e124bcc49b \
-    --hash=sha256:4f97b31b4c4e21ff58c6f330235ff893cc81e23da081b1a4b1c982075e0ed4e9 \
-    --hash=sha256:543ad8459bc438efc46d29a759e1079436290bd583141384c6f7a1068ed6f992 \
-    --hash=sha256:6a276190309aba7bc9d5bd2933230458b3521a4317acfefe69a354f2fe59f2bc \
-    --hash=sha256:73eeed32e724ea3568bb06161cad5fa7751e45bc2228e33dcb10c614044165c7 \
-    --hash=sha256:74de649d1d2ccb778f7c3afff6085bd5092aed4c23df9feeb45dd6b16f3811aa \
-    --hash=sha256:84e314d22231357d473eabec709d0ba285fa706a72377f9cc8e1cb3c8013813b \
-    --hash=sha256:9386d3ca9c145b5539a1cfc75df07757dff870168c959b473a0bccbc3abc8c73 \
-    --hash=sha256:9736ba3c85129d72aefa21b4f3bd715bc4190fe4426715abfff90481e7d00812 \
-    --hash=sha256:9f3a76670b263dc41d0ae877f09124ab96ce10e4e48f3e3e4257273cee61ad0d \
-    --hash=sha256:a1880dd6772b685e803011a6b43a230c23b566859a6e0c9a276c1e0faf4f4052 \
-    --hash=sha256:acb7564204d3c40babf93a05624fc6a8ec1ab1def295c363afc40b0c9e66c191 \
-    --hash=sha256:ad514dbfcffe30124ce655d72771ae070f30bf850b48bc4d9d3b25993ee0e386 \
-    --hash=sha256:aebc13a11ed3032d8dd6e7171eb6e86d40d67a5639d96c35142bd568b9299324 \
-    --hash=sha256:b516dad76f258a702f7ca0250885fc93d1fa5ac13ad51258e39d402bd9e2e1e4 \
-    --hash=sha256:b76130d835261b38f14fc41fdfb39ad8d672afb84c447126b84d5472244cfaba \
-    --hash=sha256:ba17845efe3aa358ec266cf9cc2800fa73038211fb27968bfa88acd09261a470 \
-    --hash=sha256:c0a03da7f2758645d17b7b4f83c8bffeae5bbb7f974523fe901f36288d2eab71 \
-    --hash=sha256:c52f81aa6f6575058d8e2c782bf79d4f9fdc89887f16825ec3a66607a5dd8e30 \
-    --hash=sha256:d4b3d2a34780645bed6414e22dda55a92e0fcd1b8a637fba86800ad737057e33 \
-    --hash=sha256:d4f13eee18433f99adefaeb7e01d83b59f73360c231d4782d9ddfaf1c3fbde0a \
-    --hash=sha256:d6cf5c05f3cee251d80e98726b5c7cc9f21bab9e9783673bac58e6dfab57ecc8 \
-    --hash=sha256:da31fbca07c435be88a0c321402c4e31a2ba61593ec7473630769de8346b54ee \
-    --hash=sha256:e21488d5cfd3d8b500b3238a6c4b075efabc18f0f6d80b29239737ebd69caa6c \
-    --hash=sha256:e31e9417ba9c42627574bdbfeada7217ad8a4cbbe45b9d6bdd4b62abbca4c6f6 \
-    --hash=sha256:eaeabf638408de2772ce3d7793b2668d4bb93807deed1725413b70e3156a7854 \
-    --hash=sha256:f266a2c0fc31995a06ebd30bcfdb7f615d7278035ec5b1cd71c48d56daaf30b0 \
-    --hash=sha256:f39a2e0ed32a0970e4e46c262753417a60c43a3246972cfc2d3eb85aedd01b21 \
-    --hash=sha256:f591704ac05dfd0477bb8f8e0bd4b5dc52c1cadf50503858dce3a15db6e46ff2 \
-    --hash=sha256:f96bd502cb11abb08efea6dab09c003305161cb6c9eafd432e35e76e7fa9b90c
+pyarrow==19.0.0 \
+    --hash=sha256:239ca66d9a05844bdf5af128861af525e14df3c9591bcc05bac25918e650d3a2 \
+    --hash=sha256:2795064647add0f16563e57e3d294dbfc067b723f0fd82ecd80af56dad15f503 \
+    --hash=sha256:29cd86c8001a94f768f79440bf83fee23963af5e7bc68ce3a7e5f120e17edf89 \
+    --hash=sha256:2a0144a712d990d60f7f42b7a31f0acaccf4c1e43e957f7b1ad58150d6f639c1 \
+    --hash=sha256:2a1a109dfda558eb011e5f6385837daffd920d54ca00669f7a11132d0b1e6042 \
+    --hash=sha256:2b6d3ce4288793350dc2d08d1e184fd70631ea22a4ff9ea5c4ff182130249d9b \
+    --hash=sha256:2f672f5364b2d7829ef7c94be199bb88bf5661dd485e21d2d37de12ccb78a136 \
+    --hash=sha256:3c1c162c4660e0978411a4761f91113dde8da3433683efa473501254563dcbe8 \
+    --hash=sha256:450a7d27e840e4d9a384b5c77199d489b401529e75a3b7a3799d4cd7957f2f9c \
+    --hash=sha256:4624c89d6f777c580e8732c27bb8e77fd1433b89707f17c04af7635dd9638351 \
+    --hash=sha256:4d8b0c0de0a73df1f1bf439af1b60f273d719d70648e898bc077547649bb8352 \
+    --hash=sha256:5418d4d0fab3a0ed497bad21d17a7973aad336d66ad4932a3f5f7480d4ca0c04 \
+    --hash=sha256:597360ffc71fc8cceea1aec1fb60cb510571a744fffc87db33d551d5de919bec \
+    --hash=sha256:5e8a28b918e2e878c918f6d89137386c06fe577cd08d73a6be8dafb317dc2d73 \
+    --hash=sha256:62ef8360ff256e960f57ce0299090fb86423afed5e46f18f1225f960e05aae3d \
+    --hash=sha256:66732e39eaa2247996a6b04c8aa33e3503d351831424cdf8d2e9a0582ac54b34 \
+    --hash=sha256:718947fb6d82409013a74b176bf93e0f49ef952d8a2ecd068fecd192a97885b7 \
+    --hash=sha256:8d47c691765cf497aaeed4954d226568563f1b3b74ff61139f2d77876717084b \
+    --hash=sha256:8e3a839bf36ec03b4315dc924d36dcde5444a50066f1c10f8290293c0427b46a \
+    --hash=sha256:9348a0137568c45601b031a8d118275069435f151cbb77e6a08a27e8125f59d4 \
+    --hash=sha256:a08e2a8a039a3f72afb67a6668180f09fddaa38fe0d21f13212b4aba4b5d2451 \
+    --hash=sha256:a218670b26fb1bc74796458d97bcab072765f9b524f95b2fccad70158feb8b17 \
+    --hash=sha256:a22a4bc0937856263df8b94f2f2781b33dd7f876f787ed746608e06902d691a5 \
+    --hash=sha256:a7bbe7109ab6198688b7079cbad5a8c22de4d47c4880d8e4847520a83b0d1b68 \
+    --hash=sha256:a92aff08e23d281c69835e4a47b80569242a504095ef6a6223c1f6bb8883431d \
+    --hash=sha256:b34d3bde38eba66190b215bae441646330f8e9da05c29e4b5dd3e41bde701098 \
+    --hash=sha256:b903afaa5df66d50fc38672ad095806443b05f202c792694f3a604ead7c6ea6e \
+    --hash=sha256:be686bf625aa7b9bada18defb3a3ea3981c1099697239788ff111d87f04cd263 \
+    --hash=sha256:c0423393e4a07ff6fea08feb44153302dd261d0551cc3b538ea7a5dc853af43a \
+    --hash=sha256:c318eda14f6627966997a7d8c374a87d084a94e4e38e9abbe97395c215830e0c \
+    --hash=sha256:c3b78eff5968a1889a0f3bc81ca57e1e19b75f664d9c61a42a604bf9d8402aae \
+    --hash=sha256:c73268cf557e688efb60f1ccbc7376f7e18cd8e2acae9e663e98b194c40c1a2d \
+    --hash=sha256:c751c1c93955b7a84c06794df46f1cec93e18610dcd5ab7d08e89a81df70a849 \
+    --hash=sha256:ce42275097512d9e4e4a39aade58ef2b3798a93aa3026566b7892177c266f735 \
+    --hash=sha256:cf3bf0ce511b833f7bc5f5bb3127ba731e97222023a444b7359f3a22e2a3b463 \
+    --hash=sha256:da410b70a7ab8eb524112f037a7a35da7128b33d484f7671a264a4c224ac131d \
+    --hash=sha256:e675a3ad4732b92d72e4d24009707e923cab76b0d088e5054914f11a797ebe44 \
+    --hash=sha256:e82c3d5e44e969c217827b780ed8faf7ac4c53f934ae9238872e749fa531f7c9 \
+    --hash=sha256:edfe6d3916e915ada9acc4e48f6dafca7efdbad2e6283db6fd9385a1b23055f1 \
+    --hash=sha256:f094742275586cdd6b1a03655ccff3b24b2610c3af76f810356c4c71d24a2a6c \
+    --hash=sha256:f208c3b58a6df3b239e0bb130e13bc7487ed14f39a9ff357b6415e3f6339b560 \
+    --hash=sha256:f43f5aef2a13d4d56adadae5720d1fed4c1356c993eda8b59dace4b5983843c1
     # via -r requirements.in
 pyasn1==0.4.8 \
     --hash=sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d \
@@ -1943,9 +1939,9 @@ requests-oauthlib==1.3.1 \
     --hash=sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5 \
     --hash=sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a
     # via google-auth-oauthlib
-responses==0.25.3 \
-    --hash=sha256:521efcbc82081ab8daa588e08f7e8a64ce79b91c39f6e62199b19159bea7dbcb \
-    --hash=sha256:617b9247abd9ae28313d57a75880422d55ec63c29d33d629697590a034358dba
+responses==0.25.6 \
+    --hash=sha256:9cac8f21e1193bb150ec557875377e41ed56248aed94e4567ed644db564bacf1 \
+    --hash=sha256:eae7ce61a9603004e76c05691e7c389e59652d91e94b419623c12bbfb8e331d8
     # via -r requirements.in
 rich==12.6.0 \
     --hash=sha256:a4eb26484f2c82589bd9a17c73d32a010b1e29d89f1604cd9bf3a2097b81bb5e \
@@ -2179,12 +2175,12 @@ stripe==6.4.0 \
     --hash=sha256:d1ab2739aa6511f34ba77f66fca9f86cffe2dbb17df84bd224a38e7254efe61b \
     --hash=sha256:d99e97e6d3f6dbaf9361ebd07b3024ec6f9d7b1f77724c8127338e0754314e9d
     # via -r requirements.in
-symbolic==12.12.4 \
-    --hash=sha256:358961063d90d368485b75df178722f984424401b92b6e56e9587da0814387e8 \
-    --hash=sha256:39c8bebeedd497cf4c6ab4844a382687cb95fe9915613b37ddc053cd5c45dd42 \
-    --hash=sha256:afcc276195be07045ec7f5f1a32f5542120833fac5c39d067dfe500387f5643a \
-    --hash=sha256:d71ad7e681d69a193ef8e71553356844eeb88abeab06e48a3b20ad2c1cf38d2c \
-    --hash=sha256:e9951ad949e235089571497e732b1bf04f93605d43e06cb352a4dce45b7a8b7c
+symbolic==12.13.3 \
+    --hash=sha256:0517fbb15b8d1b4e261ecbc35fecdc16bd35b563a7230d70d7e0091c073ecae1 \
+    --hash=sha256:2cb76190abb134e030487d93d75a1011a873fe70162492aa048f383d5e4eb914 \
+    --hash=sha256:39475cbc0ed1dfb0dd183428b7d7cb4ee85b59fac7b101a0a04bd54bbde010e9 \
+    --hash=sha256:3f0d83d7260e88bfe05f359e3a972833b1490f7b20a65e628e8df8c61c133d70 \
+    --hash=sha256:79806c75a854447ab770ff84e43daae4f73445963b13df043cbb871ce664b3be
     # via -r requirements.in
 toml==0.10.2 \
     --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \
@@ -2234,13 +2230,13 @@ types-python-dateutil==2.9.0.20241206 \
     --hash=sha256:18f493414c26ffba692a72369fea7a154c502646301ebfe3d56a04b3767284cb \
     --hash=sha256:e248a4bc70a486d3e3ec84d0dc30eec3a5f979d6e7ee4123ae043eedbb987f53
     # via -r requirements.in
-types-pytz==2024.2.0.20241003 \
-    --hash=sha256:3e22df1336c0c6ad1d29163c8fda82736909eb977281cb823c57f8bae07118b7 \
-    --hash=sha256:575dc38f385a922a212bac00a7d6d2e16e141132a3c955078f4a4fd13ed6cb44
+types-pytz==2024.2.0.20241221 \
+    --hash=sha256:06d7cde9613e9f7504766a0554a270c369434b50e00975b3a4a0f6eed0f2c1a9 \
+    --hash=sha256:8fc03195329c43637ed4f593663df721fef919b60a969066e22606edf0b53ad5
     # via -r requirements.in
-types-pyyaml==6.0.12.20240917 \
-    --hash=sha256:392b267f1c0fe6022952462bf5d6523f31e37f6cea49b14cee7ad634b6301570 \
-    --hash=sha256:d1405a86f9576682234ef83bcb4e6fff7c9305c8b1fbad5e0bcd4f7dbdc9c587
+types-pyyaml==6.0.12.20241230 \
+    --hash=sha256:7f07622dbd34bb9c8b264fe860a17e0efcad00d50b5f27e93984909d9363498c \
+    --hash=sha256:fa4d32565219b68e6dee5f67534c722e53c00d1cfc09c435ef04d7353e1e96e6
     # via
     #   -r requirements.in
     #   bigeye-sdk
@@ -2360,9 +2356,9 @@ urllib3==2.2.2 \
     #   requests
     #   responses
     #   types-requests
-virtualenv==20.17.1 \
-    --hash=sha256:ce3b1684d6e1a20a3e5ed36795a97dfc6af29bc3970ca8dab93e11ac6094b3c4 \
-    --hash=sha256:f8b927684efc6f1cc206c9db297a570ab9ad0e51c16fa9e45487d36d1905c058
+virtualenv==20.26.6 \
+    --hash=sha256:280aede09a2a5c317e409a00102e7077c6432c5a38f0ef938e643805a7ad2c48 \
+    --hash=sha256:7345cc5b25405607a624d8418154577459c3e0277f5466dd79c49d5e492995f2
     # via pre-commit
 watchdog==2.2.1 \
     --hash=sha256:102a60093090fc3ff76c983367b19849b7cc24ec414a43c0333680106e62aae1 \
@@ -2572,9 +2568,9 @@ pip==23.3.2 \
     --hash=sha256:5052d7889c1f9d05224cd41741acb7c5d6fa735ab34e339624a614eaaa7e7d76 \
     --hash=sha256:7fd9972f96db22c8077a1ee2691b172c8089b17a5652a44494a9ecb0d78f9149
     # via pip-tools
-setuptools==75.6.0 \
-    --hash=sha256:8199222558df7c86216af4f84c30e9b34a61d8ba19366cc914424cdbd28252f6 \
-    --hash=sha256:ce74b49e8f7110f9bf04883b730f4765b774ef3ef28f722cce7c273d253aaf7d
+setuptools==75.8.0 \
+    --hash=sha256:c5afc8f407c626b8313a86e10311dd3f661c6cd9c09d4bf8c15c0e11f9f2b0e6 \
+    --hash=sha256:e3982f444617239225d675215d51f6ba05f845d4eec313da4418fdbb56fb27e3
     # via
     #   bigeye-sdk
     #   nodeenv

sql/bigconfig.yml

@@ -8,9 +8,9 @@ saved_metric_definitions:
       threshold:
         type: CONSTANT
         upper_bound: 0
-      metric_schedule:
-        named_schedule:
-          name: default
+      schedule_frequency:
+        interval_type: MINUTES
+        interval_value: 0
       lookback:
         lookback_type: DATA_TIME
         lookback_window:
@@ -24,9 +24,9 @@ saved_metric_definitions:
       threshold:
         type: CONSTANT
         upper_bound: 0
-      metric_schedule:
-        named_schedule:
-          name: default
+      schedule_frequency:
+        interval_type: MINUTES
+        interval_value: 0
       rct_overrides:
         - submission_date
     - saved_metric_id: freshness_fail
@@ -71,9 +71,9 @@ saved_metric_definitions:
         type: CONSTANT
         lower_bound: 1
         upper_bound: 1
-      metric_schedule:
-        named_schedule:
-          name: default
+      schedule_frequency:
+        interval_type: MINUTES
+        interval_value: 0
       lookback:
         lookback_type: DATA_TIME
         lookback_window:
@@ -88,9 +88,9 @@ saved_metric_definitions:
         type: CONSTANT
         lower_bound: 2
         upper_bound: 2
-      metric_schedule:
-        named_schedule:
-          name: default
+      schedule_frequency:
+        interval_type: MINUTES
+        interval_value: 0
       lookback:
         lookback_type: DATA_TIME
         lookback_window:
@@ -105,9 +105,9 @@ saved_metric_definitions:
         type: CONSTANT
         lower_bound: 1
         upper_bound: 1
-      metric_schedule:
-        named_schedule:
-          name: default
+      schedule_frequency:
+        interval_type: MINUTES
+        interval_value: 0
       lookback:
         lookback_type: DATA_TIME
         lookback_window:

sql/moz-fx-data-shared-prod/addons/addon_events_clients/metadata.yaml

@@ -0,0 +1,15 @@
+---
+friendly_name: Addon Events (client) View
+description: |
+  A view on top of the events table pulling out addon relevant columns
+  to act as a central gate for downstream addon models.
+
+  See events documentation for more information about the fields:
+  https://firefox-source-docs.mozilla.org/toolkit/components/telemetry/collection/events.html
+
+  Addon event specific data documentation:
+  https://searchfox.org/mozilla-central/source/toolkit/components/telemetry/Events.yaml#174-336
+owners:
+  - kik@mozilla.com
+labels:
+  application: amo

sql/moz-fx-data-shared-prod/addons/addon_events_clients/view.sql

@@ -0,0 +1,36 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.addons.addon_events_clients`
+AS
+SELECT
+  submission_date,
+  `timestamp`,
+  COALESCE(
+    `moz-fx-data-shared-prod.udf.get_key`(event_map_values, 'addonId'),
+    `moz-fx-data-shared-prod.udf.get_key`(event_map_values, 'addon_id')
+  ) AS addon_id,
+  COALESCE(
+    `moz-fx-data-shared-prod.udf.get_key`(event_map_values, 'addonVersion'),
+    `moz-fx-data-shared-prod.udf.get_key`(event_map_values, 'addon_version')
+  ) AS addon_version,
+  event_string_value,
+  client_id,
+  sample_id,
+  normalized_channel,
+  country,
+  locale,
+  app_name,
+  app_version,
+  os AS os_name,
+  os_version,
+  event_timestamp,
+  event_category,
+  event_method AS method,
+  event_object AS `object`,
+  event_string_value AS `value`,
+  event_map_values,
+  event_process,
+FROM
+  `moz-fx-data-shared-prod.telemetry.events`
+WHERE
+  submission_date >= '2021-08-09'
+  AND event_category IN ("addonsManager", "addonsSearchDetection")

sql/moz-fx-data-shared-prod/addons/dataset_metadata.yaml

@@ -0,0 +1,11 @@
+friendly_name: Addons
+description: |-
+  Dataset for accessing Addons data.
+dataset_base_acl: view
+user_facing: true
+labels: {}
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential
+  - workgroup:amo/prod

sql/moz-fx-data-shared-prod/addons/search_detection/metadata.yaml

@@ -0,0 +1,11 @@
+---
+friendly_name: Addon - Search Detection
+description: |
+  A view for accessing the addons search detection aggregate.
+
+  Addon event specific data documentation:
+  https://searchfox.org/mozilla-central/source/toolkit/components/telemetry/Events.yaml#174-336
+owners:
+  - kik@mozilla.com
+labels:
+  application: amo

sql/moz-fx-data-shared-prod/addons/search_detection/view.sql

@@ -0,0 +1,7 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.addons.search_detection`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod.addons_derived.search_detection_v1`

sql/moz-fx-data-shared-prod/addons_derived/dataset_metadata.yaml

@@ -0,0 +1,17 @@
+friendly_name: Addons Derived
+description: |-
+  Derived data used for Addons related analysis.
+dataset_base_acl: derived
+user_facing: false
+labels: {}
+default_table_workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential
+  - workgroup:amo/prod
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential
+  - workgroup:amo/prod
+syndication: {}

sql/moz-fx-data-shared-prod/addons_derived/search_detection_v1/metadata.yaml

@@ -0,0 +1,30 @@
+---
+friendly_name: Addons - Search Detection
+description: |
+  Used for addons search detection and based on event_category: addonsSearchDetection.
+
+  This event category is recorded when a request for a search query is redirected to another
+  domain. This can be via the webRequest API, or through other means.
+
+  More information about this event_category:
+  https://searchfox.org/mozilla-central/source/toolkit/components/telemetry/Events.yaml#304
+owners:
+  - kik@mozilla.com
+labels:
+  application: amo
+  incremental: true
+  schedule: daily
+  table_type: aggregate
+  shredder_mitigation: true
+scheduling:
+  dag_name: bqetl_amo_stats
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false
+  clustering:
+    fields:
+      - normalized_channel
+      - app_name
+      - os_name

sql/moz-fx-data-shared-prod/addons_derived/search_detection_v1/query.sql

@@ -0,0 +1,37 @@
+SELECT
+  submission_date,
+  normalized_channel,
+  country,
+  app_name,
+  app_version,
+  os_name,
+  os_version,
+  addon_id,
+  addon_version,
+  method,
+  `object`,
+  `value`,
+  `moz-fx-data-shared-prod.udf.get_key`(event_map_values, 'from') AS from_etld,
+  `moz-fx-data-shared-prod.udf.get_key`(event_map_values, 'to') AS to_etld,
+  COUNT(*) AS event_count,
+  COUNT(DISTINCT client_id) AS unique_clients_cnt,
+FROM
+  `moz-fx-data-shared-prod.addons.addon_events_clients`
+WHERE
+  submission_date = @submission_date
+  AND event_category = "addonsSearchDetection"
+GROUP BY
+  submission_date,
+  normalized_channel,
+  country,
+  app_name,
+  app_version,
+  os_name,
+  os_version,
+  addon_id,
+  addon_version,
+  method,
+  `object`,
+  `value`,
+  from_etld,
+  to_etld

sql/moz-fx-data-shared-prod/addons_derived/search_detection_v1/schema.yaml

@@ -0,0 +1,97 @@
+# Based on telemetry.events: https://docs.telemetry.mozilla.org/concepts/pipeline/event_pipeline
+fields:
+- mode: NULLABLE
+  name: submission_date
+  type: DATE
+  description: |
+    Processing date.
+
+- mode: NULLABLE
+  name: normalized_channel
+  type: STRING
+  description: |
+    Normalized channel name of the Firefox app (for example: releaes, beta, nightly).
+
+- mode: NULLABLE
+  name: country
+  type: STRING
+  description: |
+    An ISO 3166-1 alpha-2 country code.
+
+- mode: NULLABLE
+  name: app_name
+  type: STRING
+  description: |
+    Name of the Firefox app.
+
+- mode: NULLABLE
+  name: app_version
+  type: STRING
+  description: |
+    Version of the Firefox app.
+
+- mode: NULLABLE
+  name: os_name
+  type: STRING
+  description: |
+    Name of the operating system used by the client.
+
+- mode: NULLABLE
+  name: os_version
+  type: STRING
+  description: |
+    Version of the operating system used by the client.
+
+- mode: NULLABLE
+  name: addon_id
+  type: STRING
+  description:
+    The id of the addon, if available.
+
+- mode: NULLABLE
+  name: addon_version
+  type: STRING
+  description: |
+    The version of the addon.
+
+- mode: NULLABLE
+  name: method
+  type: STRING
+  description: |
+    This describes the type of event that occurred.
+
+- mode: NULLABLE
+  name: object
+  type: STRING
+  description: |
+    The object the event occurred.
+
+- mode: NULLABLE
+  name: value
+  type: STRING
+  description: |
+    This is a user defined value, providing context for the event (Optional).
+
+- mode: NULLABLE
+  name: from_etld
+  type: STRING
+  description: |
+    The original domain (eTLD+1) of the request that matched a search endpoint.
+
+- mode: NULLABLE
+  name: to_etld
+  type: STRING
+  description: |
+    The domain (eTLD+1) of the final redirect target.
+
+- mode: NULLABLE
+  name: event_count
+  type: INTEGER
+  description: |
+    Number of events in this grouping.
+
+- mode: NULLABLE
+  name: unique_clients_cnt
+  type: INTEGER
+  description: |
+    Number of unique clients in this grouping.

sql/moz-fx-data-shared-prod/amo_prod/amo_stats_installs_v3/metadata.yaml

@@ -7,6 +7,9 @@ description: >
   pings have limitations on length of properties attached to events
   and addon_id values are sometimes too long. The AMO stats application
   looks up records in this table based on the hashed_addon_id.
+
+  The hashed_addon_id is only valid for install_stats:
+  https://searchfox.org/mozilla-central/rev/5b061cdc4d40d44988dc61aa941cfbd98e31791f/toolkit/components/telemetry/Events.yaml#233-234
 owners:
   - kik@mozilla.com
 labels:

sql/moz-fx-data-shared-prod/apple_ads_external/ios_app_campaign_stats_v1/bigconfig.yml

@@ -1,62 +1,61 @@
 type: BIGCONFIG_FILE
-
 tag_deployments:
-  - collection:
-      name: Operational Checks
-      notification_channels:
-        - slack: '#de-bigeye-triage'
-    deployments:
-      - column_selectors:
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.date
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.campaign
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.campaign_id
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.ad_group_name
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.ad_group_id
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.impressions
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.clicks
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.downloads
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.new_profiles
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.activated_profiles
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.repeat_users
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.week_4_retained_users
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.spend
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.lifetime_value
-        metrics:
-          - saved_metric_id: is_not_null
-            lookback:
-              lookback_type: DATA_TIME
-              lookback_window:
-                interval_type: DAYS
-                interval_value: 28
-            rct_overrides:
-              - date
-      - column_selectors:
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.campaign_country_code
-        metrics:
-          - saved_metric_id: is_2_char_len
-            lookback:
-              lookback_type: DATA_TIME
-              lookback_window:
-                interval_type: DAYS
-                interval_value: 28
-            rct_overrides:
-              - date
-      - column_selectors:
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.*
-        metrics:
-          - saved_metric_id: volume
-            lookback:
-              lookback_type: DATA_TIME
-              lookback_window:
-                interval_type: DAYS
-                interval_value: 28
-            rct_overrides:
-              - date
-          - saved_metric_id: freshness
-            lookback:
-              lookback_type: DATA_TIME
-              lookback_window:
-                interval_type: DAYS
-                interval_value: 28
-            rct_overrides:
-              - date
+- collection:
+    name: Operational Checks
+    notification_channels:
+    - slack: '#de-bigeye-triage'
+  deployments:
+  - column_selectors:
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.date
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.campaign
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.campaign_id
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.ad_group_name
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.ad_group_id
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.impressions
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.clicks
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.downloads
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.new_profiles
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.activated_profiles
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.repeat_users
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.week_4_retained_users
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.spend
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.lifetime_value
+    metrics:
+    - saved_metric_id: is_not_null
+      lookback:
+        lookback_window:
+          interval_type: DAYS
+          interval_value: 28
+        lookback_type: DATA_TIME
+      rct_overrides:
+      - date
+  - column_selectors:
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.campaign_country_code
+    metrics:
+    - saved_metric_id: is_2_char_len
+      lookback:
+        lookback_window:
+          interval_type: DAYS
+          interval_value: 28
+        lookback_type: DATA_TIME
+      rct_overrides:
+      - date
+  - column_selectors:
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.*
+    metrics:
+    - saved_metric_id: volume
+      lookback:
+        lookback_window:
+          interval_type: DAYS
+          interval_value: 28
+        lookback_type: DATA_TIME
+      rct_overrides:
+      - date
+    - saved_metric_id: freshness
+      lookback:
+        lookback_window:
+          interval_type: DAYS
+          interval_value: 28
+        lookback_type: DATA_TIME
+      rct_overrides:
+      - date

sql/moz-fx-data-shared-prod/default_browser_agent/dataset_metadata.yaml

@@ -0,0 +1,16 @@
+friendly_name: Default Browser Agent
+description: |-
+  Data related to default browser agent
+dataset_base_acl: view
+user_facing: true
+labels: {}
+default_table_workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential
+default_table_expiration_ms: null
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential
+syndication: {}

sql/moz-fx-data-shared-prod/default_browser_agent/default_browser_agg/view.sql

@@ -0,0 +1,7 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.default_browser_agent.default_browser_agg`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod.default_browser_agent_derived.default_browser_agg_v1`

sql/moz-fx-data-shared-prod/default_browser_agent/default_browser_agg_by_os/view.sql

@@ -0,0 +1,7 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.default_browser_agent.default_browser_agg_by_os`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod.default_browser_agent_derived.default_browser_agg_by_os_v1`

sql/moz-fx-data-shared-prod/default_browser_agent_derived/dataset_metadata.yaml

@@ -0,0 +1,16 @@
+friendly_name: Default Browser Agent Derived
+description: |-
+  Data related to default browser agent
+dataset_base_acl: derived
+user_facing: false
+labels: {}
+default_table_workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential
+default_table_expiration_ms: null
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential
+syndication: {}

sql/moz-fx-data-shared-prod/default_browser_agent_derived/default_browser_agg_by_os_v1/backfill.yaml

@@ -0,0 +1,9 @@
+2024-12-27:
+  start_date: 2023-01-01
+  end_date: 2023-12-31
+  reason: https://mozilla-hub.atlassian.net/browse/DENG-6889
+  watchers:
+  - wichan@mozilla.com
+  status: Complete
+  shredder_mitigation: false
+  override_retention_limit: false

sql/moz-fx-data-shared-prod/default_browser_agent_derived/default_browser_agg_by_os_v1/metadata.yaml

@@ -0,0 +1,23 @@
+friendly_name: Default Browser Aggregates Derived By OS version
+description: |-
+  Aggregate table for default browser setting by os version
+owners:
+- wichan@mozilla.com
+labels:
+  incremental: true
+  owner1: wichan@mozilla.com
+  table_type: aggregate
+  shredder_mitigation: true
+scheduling:
+  dag_name: bqetl_default_browser_aggregates
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: true
+    expiration_days: null
+  range_partitioning: null
+  clustering:
+    fields:
+    - os_version
+references: {}

sql/moz-fx-data-shared-prod/default_browser_agent_derived/default_browser_agg_by_os_v1/query.sql

@@ -0,0 +1,19 @@
+SELECT
+  DATE(submission_timestamp) AS submission_date,
+  os_version,
+  mozfun.norm.get_windows_info(`os_version`).name AS windows_name,
+  default_browser,
+  previous_default_browser,
+  mozfun.norm.truncate_version(build_version, 'major') AS build_version_major,
+  COUNT(*) AS row_count
+FROM
+  `moz-fx-data-shared-prod.default_browser_agent.default_browser`
+WHERE
+  DATE(submission_timestamp) = @submission_date
+GROUP BY
+  DATE(submission_timestamp),
+  os_version,
+  mozfun.norm.get_windows_info(`os_version`).name,
+  default_browser,
+  previous_default_browser,
+  mozfun.norm.truncate_version(build_version, 'major')

sql/moz-fx-data-shared-prod/default_browser_agent_derived/default_browser_agg_by_os_v1/schema.yaml

@@ -0,0 +1,29 @@
+fields:
+- name: submission_date
+  type: DATE
+  mode: NULLABLE
+  description: Date when the ingestion edge server accepted this message.
+- name: os_version
+  type: STRING
+  mode: NULLABLE
+  description: Windows version number in major.minor.build.UBR format (UBR is optional, only available on Win10).
+- name: windows_name
+  type: STRING
+  mode: NULLABLE
+  description: Name of the window os version.
+- name: default_browser
+  type: STRING
+  mode: NULLABLE
+  description: The current default browser.
+- name: previous_default_browser
+  type: STRING
+  mode: NULLABLE
+  description: What the default browser was before it was changed to the current one.
+- name: build_version_major
+  type: NUMERIC
+  mode: NULLABLE
+  description: Major build version.
+- name: row_count
+  type: INTEGER
+  mode: NULLABLE
+  description: The total number of rows aggregated.

sql/moz-fx-data-shared-prod/default_browser_agent_derived/default_browser_agg_v1/backfill.yaml

@@ -0,0 +1,9 @@
+2024-12-27:
+  start_date: 2023-01-01
+  end_date: 2023-12-31
+  reason: https://mozilla-hub.atlassian.net/browse/DENG-6889
+  watchers:
+  - wichan@mozilla.com
+  status: Complete
+  shredder_mitigation: false
+  override_retention_limit: false

sql/moz-fx-data-shared-prod/default_browser_agent_derived/default_browser_agg_v1/metadata.yaml

@@ -0,0 +1,23 @@
+friendly_name: Default Browser Aggregates Derived
+description: |-
+  Aggregate table for default browser setting
+owners:
+- wichan@mozilla.com
+labels:
+  incremental: true
+  owner1: wichan@mozilla.com
+  table_type: aggregate
+  shredder_mitigation: true
+scheduling:
+  dag_name: bqetl_default_browser_aggregates
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: true
+    expiration_days: null
+  range_partitioning: null
+  clustering:
+    fields:
+    - normalized_country_code
+references: {}

sql/moz-fx-data-shared-prod/default_browser_agent_derived/default_browser_agg_v1/query.sql

@@ -0,0 +1,27 @@
+SELECT
+  DATE(submission_timestamp) AS submission_date,
+  mozfun.norm.truncate_version(build_version, 'major') AS build_version_major,
+  mozfun.norm.get_windows_info(`os_version`).name AS windows_name,
+  normalized_country_code,
+  build_channel,
+  default_browser,
+  previous_default_browser,
+  notification_shown,
+  notification_type,
+  notification_action,
+  COUNT(*) AS row_count
+FROM
+  `moz-fx-data-shared-prod.default_browser_agent.default_browser`
+WHERE
+  DATE(submission_timestamp) = @submission_date
+GROUP BY
+  DATE(submission_timestamp),
+  mozfun.norm.truncate_version(build_version, 'major'),
+  mozfun.norm.get_windows_info(`os_version`).name,
+  normalized_country_code,
+  build_channel,
+  default_browser,
+  previous_default_browser,
+  notification_shown,
+  notification_type,
+  notification_action

sql/moz-fx-data-shared-prod/default_browser_agent_derived/default_browser_agg_v1/schema.yaml

@@ -0,0 +1,46 @@
+fields:
+- name: submission_date
+  type: DATE
+  mode: NULLABLE
+  description: Date when the ingestion edge server accepted this message.
+- name: build_version_major
+  type: NUMERIC
+  mode: NULLABLE
+  description: Major build version.
+- name: windows_name
+  type: STRING
+  mode: NULLABLE
+  description: Name of the window os version.
+- name: normalized_country_code
+  type: STRING
+  mode: NULLABLE
+  description: An ISO 3166-1 alpha-2 country code.
+- name: build_channel
+  type: STRING
+  mode: NULLABLE
+  description: build channel.
+- name: default_browser
+  type: STRING
+  mode: NULLABLE
+  description: The current default browser.
+- name: previous_default_browser
+  type: STRING
+  mode: NULLABLE
+  description: What the default browser was before it was changed to the current one.
+- name: notification_shown
+  type: STRING
+  mode: NULLABLE
+  description: Whether the notification was shown, not shown, or resulted in an error.
+- name: notification_type
+  type: STRING
+  mode: NULLABLE
+  description: Whether the notification shown was the initial notification, or the
+    followup.
+- name: notification_action
+  type: STRING
+  mode: NULLABLE
+  description: What action was taken by the user then the toast notification was displayed.
+- name: row_count
+  type: INTEGER
+  mode: NULLABLE
+  description: The total number of rows aggregated.

sql/moz-fx-data-shared-prod/fenix/locale_aggregates/view.sql

@@ -0,0 +1,7 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.fenix.locale_aggregates`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod.fenix_derived.locale_aggregates_v1`

sql/moz-fx-data-shared-prod/fenix_derived/feature_usage_metrics_v2/metadata.yaml

@@ -0,0 +1,20 @@
+friendly_name: Feature Usage Metrics (Android - Fenix)
+description: |-
+  Metrics from metrics pings for mobile feature usage dashboards (Fenix)
+owners:
+- rzhao@mozilla.com
+- vsabino@mozilla.com
+labels:
+  application: fenix
+  incremental: true
+  owner1: rzhao
+  table_type: aggregate
+  shredder_mitigation: true
+scheduling:
+  dag_name: bqetl_mobile_feature_usage
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: true
+    expiration_days: null

sql/moz-fx-data-shared-prod/fenix_derived/feature_usage_metrics_v2/query.sql

@@ -0,0 +1,331 @@
+WITH baseline_clients AS (
+  SELECT
+    submission_date AS dau_date,
+    client_id
+  FROM
+    `moz-fx-data-shared-prod.fenix.baseline_clients_daily`
+  WHERE
+    submission_date = DATE_SUB(@submission_date, INTERVAL 4 DAY)
+    AND durations > 0
+    AND LOWER(COALESCE(isp, "")) <> "browserstack"
+    AND LOWER(COALESCE(distribution_id, "")) <> "mozillaonline"
+),
+client_attribution AS (
+  SELECT
+    client_id,
+    adjust_network,
+  FROM
+    `moz-fx-data-shared-prod.fenix.attribution_clients`
+),
+metrics_dau AS (
+  -- assign a DAU date for each metric ping while keeping it de-duplicated
+  SELECT
+    m.*,
+    MAX(dau_date) OVER (PARTITION BY document_id) AS dau_date
+  FROM
+    `moz-fx-data-shared-prod.fenix.metrics` m
+  JOIN
+    baseline_clients
+    ON client_info.client_id = client_id
+     -- offset by at least one to reflect metrics ping design considerations
+    AND DATE_DIFF(DATE(submission_timestamp), dau_date, DAY)
+    BETWEEN 1
+    AND 4
+  WHERE
+    DATE(submission_timestamp)
+    BETWEEN DATE_SUB(@submission_date, INTERVAL 3 DAY)
+    AND @submission_date
+  QUALIFY
+    ROW_NUMBER() OVER (PARTITION BY document_id ORDER BY submission_timestamp) = 1
+),
+metric_ping_clients_feature_usage AS (
+  SELECT
+    dau_date,
+    client_info.client_id,
+    ARRAY_AGG(normalized_channel ORDER BY submission_timestamp DESC)[SAFE_OFFSET(0)] AS channel,
+    ARRAY_AGG(normalized_country_code ORDER BY submission_timestamp DESC)[
+      SAFE_OFFSET(0)
+    ] AS country,
+    LOGICAL_OR(COALESCE(metrics.boolean.metrics_default_browser, FALSE)) AS is_default_browser,
+    --Credential Management: Logins
+    SUM(COALESCE(metrics.counter.logins_deleted, 0)) AS logins_deleted,
+    SUM(COALESCE(metrics.counter.logins_modified, 0)) AS logins_modified,
+    MAX(COALESCE(metrics.quantity.logins_saved_all, 0)) AS currently_stored_logins,
+    --Credential Management: Credit Cards
+    SUM(COALESCE(metrics.counter.credit_cards_deleted, 0)) AS credit_cards_deleted,
+    MAX(COALESCE(metrics.quantity.credit_cards_saved_all, 0)) AS currently_stored_credit_cards,
+    --Credential Management: Addresses
+    SUM(COALESCE(metrics.counter.addresses_deleted, 0)) AS addresses_deleted,
+    SUM(COALESCE(metrics.counter.addresses_updated, 0)) AS addresses_modified,
+    MAX(COALESCE(metrics.quantity.addresses_saved_all, 0)) AS currently_stored_addresses,
+    --Bookmark
+    SUM(COALESCE(metrics_bookmarks_add_table.value, 0)) AS bookmarks_add,
+    SUM(COALESCE(metrics_bookmarks_delete_table.value, 0)) AS bookmarks_delete,
+    SUM(COALESCE(metrics_bookmarks_edit_table.value, 0)) AS bookmarks_edit,
+    SUM(COALESCE(metrics_bookmarks_open_table.value, 0)) AS bookmarks_open,
+    MAX(
+      COALESCE(metrics.counter.metrics_desktop_bookmarks_count, 0)
+    ) AS metrics_desktop_bookmarks_count,
+    MAX(
+      COALESCE(metrics.counter.metrics_mobile_bookmarks_count, 0)
+    ) AS metrics_mobile_bookmarks_count,
+    CAST(
+      MAX(COALESCE(metrics.boolean.metrics_has_desktop_bookmarks, FALSE)) AS INT64
+    ) AS metrics_has_desktop_bookmarks,
+    CAST(
+      MAX(COALESCE(metrics.boolean.metrics_has_mobile_bookmarks, FALSE)) AS INT64
+    ) AS metrics_has_mobile_bookmarks,
+    --Privacy
+    CAST(
+      MAX(
+        COALESCE(LOWER(metrics.string.preferences_enhanced_tracking_protection) = "standard", FALSE)
+      ) AS INT64
+    ) AS etp_standard,
+    CAST(
+      MAX(
+        COALESCE(LOWER(metrics.string.preferences_enhanced_tracking_protection) = "strict", FALSE)
+      ) AS INT64
+    ) AS etp_strict,
+    CAST(
+      MAX(
+        COALESCE(LOWER(metrics.string.preferences_enhanced_tracking_protection) = "custom", FALSE)
+      ) AS INT64
+    ) AS etp_custom,
+    CAST(
+      MAX(
+        COALESCE(
+          LOWER(metrics.string.preferences_enhanced_tracking_protection) NOT IN (
+            'custom',
+            'standard',
+            'strict'
+          ),
+          FALSE
+        )
+      ) AS INT64
+    ) AS etp_disabled,
+    --Tab count
+    MAX(
+      COALESCE(metrics.counter.metrics_private_tabs_open_count, 0)
+    ) AS metrics_private_tabs_open_count,
+    MAX(COALESCE(metrics.counter.metrics_tabs_open_count, 0)) AS metrics_tabs_open_count,
+    --Default browser
+    CAST(
+      MAX(COALESCE(metrics.boolean.metrics_default_browser, FALSE)) AS INT64
+    ) AS metrics_default_browser,
+    --Awesomebar Location
+    CAST(
+      MAX(
+        COALESCE(metrics.string.preferences_toolbar_position_setting IN ('top', 'fixed_top'), FALSE)
+      ) AS INT64
+    ) AS awesomebar_top,
+    CAST(
+      MAX(COALESCE(metrics.string.preferences_toolbar_position_setting = 'bottom', FALSE)) AS INT64
+    ) AS awesomebar_bottom,
+    CAST(
+      MAX(
+        COALESCE(
+          LOWER(metrics.string.preferences_toolbar_position_setting) NOT IN (
+            'top',
+            'fixed_top',
+            'bottom'
+          ),
+          FALSE
+        )
+      ) AS INT64
+    ) AS awesomebar_null,
+    --Notification
+    CAST(
+      MAX(COALESCE(metrics.boolean.metrics_notifications_allowed, FALSE)) AS INT64
+    ) AS metrics_notifications_allowed,
+    CAST(
+      MAX(COALESCE(metrics.boolean.events_marketing_notification_allowed, FALSE)) AS INT64
+    ) AS events_marketing_notification_allowed,
+    --Customize Homepage
+    CAST(
+      MAX(COALESCE(metrics.boolean.customize_home_contile, FALSE)) AS INT64
+    ) AS customize_home_contile,
+    CAST(
+      MAX(COALESCE(metrics.boolean.customize_home_jump_back_in, FALSE)) AS INT64
+    ) AS customize_home_jump_back_in,
+    CAST(
+      MAX(COALESCE(metrics.boolean.customize_home_most_visited_sites, FALSE)) AS INT64
+    ) AS customize_home_most_visited_sites,
+    CAST(
+      MAX(COALESCE(metrics.boolean.customize_home_pocket, FALSE)) AS INT64
+    ) AS customize_home_pocket,
+    CAST(
+      MAX(COALESCE(metrics.boolean.customize_home_recently_saved, FALSE)) AS INT64
+    ) AS customize_home_recently_saved,
+    CAST(
+      MAX(COALESCE(metrics.boolean.customize_home_recently_visited, FALSE)) AS INT64
+    ) AS customize_home_recently_visited
+  FROM
+    metrics_dau
+  LEFT JOIN
+    UNNEST(metrics.labeled_counter.metrics_bookmarks_add) AS metrics_bookmarks_add_table
+  LEFT JOIN
+    UNNEST(metrics.labeled_counter.metrics_bookmarks_delete) AS metrics_bookmarks_delete_table
+  LEFT JOIN
+    UNNEST(metrics.labeled_counter.metrics_bookmarks_edit) AS metrics_bookmarks_edit_table
+  LEFT JOIN
+    UNNEST(metrics.labeled_counter.metrics_bookmarks_open) AS metrics_bookmarks_open_table
+  GROUP BY
+    dau_date,
+    client_id
+)
+-- Aggregated feature usage
+SELECT
+  @submission_date AS submission_date,
+  dau_date AS metric_date,
+  COUNT(DISTINCT client_id) AS clients,
+  channel,
+  country,
+  adjust_network,
+  is_default_browser,
+  /*Logins*/
+  --logins deleted
+  COUNT(DISTINCT IF(logins_deleted > 0, client_id, NULL)) AS logins_deleted_users,
+  SUM(logins_deleted) AS logins_deleted,
+  --logins modified
+  COUNT(DISTINCT IF(logins_modified > 0, client_id, NULL)) AS logins_modified_users,
+  SUM(logins_modified) AS logins_modified,
+  --logins currently stored
+  COUNT(DISTINCT IF(currently_stored_logins > 0, client_id, NULL)) AS currently_stored_logins_users,
+  SUM(currently_stored_logins) AS currently_stored_logins,
+  /*Credit Card*/
+  --credit card deleted
+  COUNT(DISTINCT IF(credit_cards_deleted > 0, client_id, NULL)) AS credit_cards_deleted_users,
+  SUM(credit_cards_deleted) AS credit_cards_deleted,
+  --credit card currently stored
+  COUNT(
+    DISTINCT IF(currently_stored_credit_cards > 0, client_id, NULL)
+  ) AS currently_stored_credit_cards_users,
+  SUM(currently_stored_credit_cards) AS currently_stored_credit_cards,
+  /*Address*/
+  --address deleted
+  COUNT(DISTINCT IF(addresses_deleted > 0, client_id, NULL)) AS addresses_deleted_users,
+  SUM(addresses_deleted) AS addresses_deleted,
+  --address modified
+  COUNT(DISTINCT IF(addresses_modified > 0, client_id, NULL)) AS addresses_modified_users,
+  SUM(addresses_modified) AS addresses_modified,
+  -- addresses currently stored
+  COUNT(
+    DISTINCT IF(currently_stored_addresses > 0, client_id, NULL)
+  ) AS currently_stored_addresses_users,
+  SUM(currently_stored_addresses) AS currently_stored_addresses,
+  /*Bookmark*/
+  --bookmarks_add
+  COUNT(DISTINCT IF(bookmarks_add > 0, client_id, NULL)) AS bookmarks_add_users,
+  SUM(bookmarks_add) AS bookmarks_add,
+  --bookmarks_delete
+  COUNT(DISTINCT IF(bookmarks_delete > 0, client_id, NULL)) AS bookmarks_delete_users,
+  SUM(bookmarks_delete) AS bookmarks_delete,
+  --bookmarks_edit
+  COUNT(DISTINCT IF(bookmarks_edit > 0, client_id, NULL)) AS bookmarks_edit_users,
+  SUM(bookmarks_edit) AS bookmarks_edit,
+  --bookmarks_open
+  COUNT(DISTINCT IF(bookmarks_open > 0, client_id, NULL)) AS bookmarks_open_users,
+  SUM(bookmarks_open) AS bookmarks_open,
+  --metrics_desktop_bookmarks_count
+  COUNT(
+    DISTINCT IF(metrics_desktop_bookmarks_count > 0, client_id, NULL)
+  ) AS metrics_desktop_bookmarks_count_users,
+  SUM(metrics_desktop_bookmarks_count) AS metrics_desktop_bookmarks_count,
+  --metrics_mobile_bookmarks_count
+  COUNT(
+    DISTINCT IF(metrics_mobile_bookmarks_count > 0, client_id, NULL)
+  ) AS metrics_mobile_bookmarks_count_users,
+  SUM(metrics_mobile_bookmarks_count) AS metrics_mobile_bookmarks_count,
+  --metrics_has_desktop_bookmarks
+  COUNT(
+    DISTINCT IF(metrics_has_desktop_bookmarks > 0, client_id, NULL)
+  ) AS metrics_has_desktop_bookmarks_users,
+  SUM(metrics_has_desktop_bookmarks) AS metrics_has_desktop_bookmarks,
+  --metrics_has_mobile_bookmarks
+  COUNT(
+    DISTINCT IF(metrics_has_mobile_bookmarks > 0, client_id, NULL)
+  ) AS metrics_has_mobile_bookmarks_users,
+  SUM(metrics_has_mobile_bookmarks) AS metrics_has_mobile_bookmarks,
+  /*Privacy*/
+  --etp_standard
+  COUNT(DISTINCT IF(etp_standard > 0, client_id, NULL)) AS etp_standard_users,
+  SUM(etp_standard) AS etp_standard,
+  --etp_strict
+  COUNT(DISTINCT IF(etp_strict > 0, client_id, NULL)) AS etp_strict_users,
+  SUM(etp_strict) AS etp_strict,
+  --etp_custom
+  COUNT(DISTINCT IF(etp_custom > 0, client_id, NULL)) AS etp_custom_users,
+  SUM(etp_custom) AS etp_custom,
+    --etp_disabled
+  COUNT(DISTINCT IF(etp_disabled > 0, client_id, NULL)) AS etp_disabled_users,
+  SUM(etp_disabled) AS etp_disabled,
+  /*Tab Count*/
+  --metrics_private_tabs_open_count
+  COUNT(
+    DISTINCT IF(metrics_private_tabs_open_count > 0, client_id, NULL)
+  ) AS metrics_private_tabs_open_count_users,
+  SUM(metrics_private_tabs_open_count) AS metrics_private_tabs_open_count,
+  --metrics_tabs_open_count
+  COUNT(DISTINCT IF(metrics_tabs_open_count > 0, client_id, NULL)) AS metrics_tabs_open_count_users,
+  SUM(metrics_tabs_open_count) AS metrics_tabs_open_count,
+  /*Default Browser*/
+  --metrics_default_browser
+  COUNT(DISTINCT IF(metrics_default_browser > 0, client_id, NULL)) AS metrics_default_browser_users,
+  SUM(metrics_default_browser) AS metrics_default_browser,
+  /*Awesomebar Location*/
+  --awesomebar_top
+  COUNT(DISTINCT IF(awesomebar_top > 0, client_id, NULL)) AS awesomebar_top_users,
+  --awesomebar_bottom
+  COUNT(DISTINCT IF(awesomebar_bottom > 0, client_id, NULL)) AS awesomebar_bottom_users,
+  --awesomebar_null
+  COUNT(DISTINCT IF(awesomebar_null > 0, client_id, NULL)) AS awesomebar_null_users,
+  /*Notificaion*/
+  --metrics_notifications_allowed
+  COUNT(
+    DISTINCT IF(metrics_notifications_allowed > 0, client_id, NULL)
+  ) AS metrics_notifications_allowed_users,
+  SUM(metrics_notifications_allowed) AS metrics_notifications_allowed,
+  --events_marketing_notification_allowed
+  COUNT(
+    DISTINCT IF(events_marketing_notification_allowed > 0, client_id, NULL)
+  ) AS events_marketing_notification_allowed_users,
+  SUM(events_marketing_notification_allowed) AS events_marketing_notification_allowed,
+  /*Customize Homepage*/
+  --customize_home_contile
+  COUNT(DISTINCT IF(customize_home_contile > 0, client_id, NULL)) AS customize_home_contile_users,
+  SUM(customize_home_contile) AS customize_home_contile,
+  --customize_home_jump_back_in_users
+  COUNT(
+    DISTINCT IF(customize_home_jump_back_in > 0, client_id, NULL)
+  ) AS customize_home_jump_back_in_users,
+  SUM(customize_home_jump_back_in) AS customize_home_jump_back_in,
+  --customize_home_most_visited_sites_users
+  COUNT(
+    DISTINCT IF(customize_home_most_visited_sites > 0, client_id, NULL)
+  ) AS customize_home_most_visited_sites_users,
+  SUM(customize_home_most_visited_sites) AS customize_home_most_visited_sites,
+  --customize_home_pocket_users
+  COUNT(DISTINCT IF(customize_home_pocket > 0, client_id, NULL)) AS customize_home_pocket_users,
+  SUM(customize_home_pocket) AS customize_home_pocket,
+  --customize_home_recently_saved_users
+  COUNT(
+    DISTINCT IF(customize_home_recently_saved > 0, client_id, NULL)
+  ) AS customize_home_recently_saved_users,
+  SUM(customize_home_recently_saved) AS customize_home_recently_saved,
+  --customize_home_recently_visited_users
+  COUNT(
+    DISTINCT IF(customize_home_recently_visited > 0, client_id, NULL)
+  ) AS customize_home_recently_visited_users,
+  SUM(customize_home_recently_visited) AS customize_home_recently_visited
+FROM
+  metric_ping_clients_feature_usage
+LEFT JOIN
+  client_attribution
+  USING (client_id)
+GROUP BY
+  submission_date,
+  metric_date,
+  channel,
+  country,
+  adjust_network,
+  is_default_browser

sql/moz-fx-data-shared-prod/fenix_derived/feature_usage_metrics_v2/schema.yaml

@@ -0,0 +1,289 @@
+fields:
+- name: submission_date
+  type: DATE
+  mode: NULLABLE
+  description: Airflow's internal execution_date.
+- name: metric_date
+  type: DATE
+  mode: NULLABLE
+  description: reporting date for feature usage, aligned with DAU.
+- name: clients
+  type: INTEGER
+  mode: NULLABLE
+  description: number of clients accounted for in the aggregation.
+- name: channel
+  type: STRING
+  mode: NULLABLE
+  description: Channel
+- name: country
+  type: STRING
+  mode: NULLABLE
+  description: Country
+- name: adjust_network
+  type: STRING
+  mode: NULLABLE
+  description: Adjust Network
+- name: is_default_browser
+  type: BOOLEAN
+  mode: NULLABLE
+  description: Is Default Browser
+- name: logins_deleted_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Logins Deleted Users
+- name: logins_deleted
+  type: INTEGER
+  mode: NULLABLE
+  description: Logins Deleted
+- name: logins_modified_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Logins Modified Users
+- name: logins_modified
+  type: INTEGER
+  mode: NULLABLE
+  description: Logins Modified
+- name: currently_stored_logins_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Currently Stored Logins Users
+- name: currently_stored_logins
+  type: INTEGER
+  mode: NULLABLE
+  description: Currently Stored Logins
+- name: credit_cards_deleted_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Credit Cards Deleted Users
+- name: credit_cards_deleted
+  type: INTEGER
+  mode: NULLABLE
+  description: Credit Cards Deleted
+- name: currently_stored_credit_cards_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Currently Stored Credit Cards Users
+- name: currently_stored_credit_cards
+  type: INTEGER
+  mode: NULLABLE
+  description: Currently Stored Credit Cards
+- name: addresses_deleted_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Addresses Deleted Users
+- name: addresses_deleted
+  type: INTEGER
+  mode: NULLABLE
+  description: Addresses Deleted
+- name: addresses_modified_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Addresses Modified Users
+- name: addresses_modified
+  type: INTEGER
+  mode: NULLABLE
+  description: Addresses Modified
+- name: currently_stored_addresses_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Currently Stored Addresses Users
+- name: currently_stored_addresses
+  type: INTEGER
+  mode: NULLABLE
+  description: Currently Stored Addresses
+- name: bookmarks_add_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Bookmarks Add Users
+- name: bookmarks_add
+  type: INTEGER
+  mode: NULLABLE
+  description: Bookmarks Add
+- name: bookmarks_delete_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Bookmarks Delete Users
+- name: bookmarks_delete
+  type: INTEGER
+  mode: NULLABLE
+  description: Bookmarks Delete
+- name: bookmarks_edit_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Bookmarks Edit Users
+- name: bookmarks_edit
+  type: INTEGER
+  mode: NULLABLE
+  description: Bookmarks Edit
+- name: bookmarks_open_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Bookmarks Open Users
+- name: bookmarks_open
+  type: INTEGER
+  mode: NULLABLE
+  description: Bookmarks Open
+- name: metrics_desktop_bookmarks_count_users
+  type: INTEGER
+  mode: NULLABLE
+  description: metrics_desktop_bookmarks_count_users
+- name: metrics_desktop_bookmarks_count
+  type: INTEGER
+  mode: NULLABLE
+  description: metrics_desktop_bookmarks_count
+- name: metrics_mobile_bookmarks_count_users
+  type: INTEGER
+  mode: NULLABLE
+  description: metrics_mobile_bookmarks_count_users
+- name: metrics_mobile_bookmarks_count
+  type: INTEGER
+  mode: NULLABLE
+  description: metrics_mobile_bookmarks_count
+- name: metrics_has_desktop_bookmarks_users
+  type: INTEGER
+  mode: NULLABLE
+  description: metrics_has_desktop_bookmarks_users
+- name: metrics_has_desktop_bookmarks
+  type: INTEGER
+  mode: NULLABLE
+  description: Metric Has Desktop Bookmarks
+- name: metrics_has_mobile_bookmarks_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Metrics Has Mobile Bookmarks Users
+- name: metrics_has_mobile_bookmarks
+  type: INTEGER
+  mode: NULLABLE
+  description: Metrics Has Mobile Bookmarks
+- name: etp_standard_users
+  type: INTEGER
+  mode: NULLABLE
+  description: ETP Standard Users
+- name: etp_standard
+  type: INTEGER
+  mode: NULLABLE
+  description: ETP Standard
+- name: etp_strict_users
+  type: INTEGER
+  mode: NULLABLE
+  description: ETP Strict Users
+- name: etp_strict
+  type: INTEGER
+  mode: NULLABLE
+  description: ETP Strict
+- name: etp_custom_users
+  type: INTEGER
+  mode: NULLABLE
+  description: ETP Custom Users
+- name: etp_custom
+  type: INTEGER
+  mode: NULLABLE
+  description: ETP Custom
+- name: etp_disabled_users
+  type: INTEGER
+  mode: NULLABLE
+  description: ETP Disabled Users
+- name: etp_disabled
+  type: INTEGER
+  mode: NULLABLE
+  description: ETP Disabled
+- name: metrics_private_tabs_open_count_users
+  type: INTEGER
+  mode: NULLABLE
+  description: metrics_private_tabs_open_count_users
+- name: metrics_private_tabs_open_count
+  type: INTEGER
+  mode: NULLABLE
+  description: metrics_private_tabs_open_count
+- name: metrics_tabs_open_count_users
+  type: INTEGER
+  mode: NULLABLE
+  description: metrics_tabs_open_count_users
+- name: metrics_tabs_open_count
+  type: INTEGER
+  mode: NULLABLE
+  description: metrics_tabs_open_count
+- name: metrics_default_browser_users
+  type: INTEGER
+  mode: NULLABLE
+  description: metrics_default_browser_users
+- name: metrics_default_browser
+  type: INTEGER
+  mode: NULLABLE
+  description: metrics_default_browser
+- name: awesomebar_top_users
+  type: INTEGER
+  mode: NULLABLE
+  description: awesomebar_top_users
+- name: awesomebar_bottom_users
+  type: INTEGER
+  mode: NULLABLE
+  description: awesomebar_bottom_users
+- name: awesomebar_null_users
+  type: INTEGER
+  mode: NULLABLE
+  description: awesomebar_null_users
+- name: metrics_notifications_allowed_users
+  type: INTEGER
+  mode: NULLABLE
+  description: metrics_notifications_allowed_users
+- name: metrics_notifications_allowed
+  type: INTEGER
+  mode: NULLABLE
+  description: metrics_notifications_allowed
+- name: events_marketing_notification_allowed_users
+  type: INTEGER
+  mode: NULLABLE
+  description: events_marketing_notification_allowed_users
+- name: events_marketing_notification_allowed
+  type: INTEGER
+  mode: NULLABLE
+  description: Events Marketing Notification Allowed
+- name: customize_home_contile_users
+  type: INTEGER
+  mode: NULLABLE
+  description: customize_home_contile_users
+- name: customize_home_contile
+  type: INTEGER
+  mode: NULLABLE
+  description: customize_home_contile
+- name: customize_home_jump_back_in_users
+  type: INTEGER
+  mode: NULLABLE
+  description: customize_home_jump_back_in_users
+- name: customize_home_jump_back_in
+  type: INTEGER
+  mode: NULLABLE
+  description: customize_home_jump_back_in
+- name: customize_home_most_visited_sites_users
+  type: INTEGER
+  mode: NULLABLE
+  description: customize_home_most_visited_sites_users
+- name: customize_home_most_visited_sites
+  type: INTEGER
+  mode: NULLABLE
+  description: customize_home_most_visited_sites
+- name: customize_home_pocket_users
+  type: INTEGER
+  mode: NULLABLE
+  description: customize_home_pocket_users
+- name: customize_home_pocket
+  type: INTEGER
+  mode: NULLABLE
+  description: customize_home_pocket
+- name: customize_home_recently_saved_users
+  type: INTEGER
+  mode: NULLABLE
+  description: customize_home_recently_saved_users
+- name: customize_home_recently_saved
+  type: INTEGER
+  mode: NULLABLE
+  description: customize_home_recently_saved
+- name: customize_home_recently_visited_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Customize Home Recently Visited Users
+- name: customize_home_recently_visited
+  type: INTEGER
+  mode: NULLABLE
+  description: Customize Home Recently Visited

sql/moz-fx-data-shared-prod/fenix_derived/locale_aggregates_v1/metadata.yaml

@@ -0,0 +1,23 @@
+friendly_name: Locale Aggregates
+description: |-
+  Fenix DAU, WAU, MAU and client counts by locale and core dimensions of analysis.
+owners:
+- lvargas@mozilla.com
+labels:
+  incremental: true
+  shredder_mitigation: true
+scheduling:
+  dag_name: bqetl_analytics_aggregations
+  date_partition_offset: -1
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: true
+    expiration_days: null
+  clustering:
+    fields:
+    - country
+    - channel
+    - locale
+references: {}

sql/moz-fx-data-shared-prod/fenix_derived/locale_aggregates_v1/query.sql

@@ -0,0 +1,98 @@
+-- Query for fenix_derived.locale_aggregates_v1
+WITH metrics AS (
+    -- Metrics ping may arrive in the same or next day as the baseline ping.
+  SELECT
+    client_id,
+    ARRAY_AGG(normalized_channel IGNORE NULLS ORDER BY submission_date ASC)[
+      SAFE_OFFSET(0)
+    ] AS normalized_channel,
+    ARRAY_AGG(is_default_browser IGNORE NULLS ORDER BY submission_date ASC)[
+      SAFE_OFFSET(0)
+    ] AS is_default_browser
+  FROM
+    `moz-fx-data-shared-prod.fenix.metrics_clients_last_seen`
+  WHERE
+    DATE(submission_date)
+    BETWEEN @submission_date
+    AND DATE_ADD(@submission_date, INTERVAL 1 DAY)
+  GROUP BY
+    client_id
+),
+baseline AS (
+  SELECT
+    client_id,
+    submission_date,
+    app_name,
+    app_display_version,
+    normalized_channel,
+    country,
+    city,
+    locale,
+    normalized_os,
+    normalized_os_version,
+    is_daily_user,
+    is_weekly_user,
+    is_monthly_user,
+    is_dau,
+    is_wau,
+    is_mau
+  FROM
+    `moz-fx-data-shared-prod.fenix.active_users`
+  WHERE
+    submission_date = @submission_date
+),
+unioned AS (
+  SELECT
+    baseline.client_id,
+    baseline.submission_date,
+    baseline.app_name,
+    baseline.app_display_version AS app_version,
+    baseline.normalized_channel AS channel,
+    IFNULL(baseline.country, '??') country,
+    baseline.city,
+    baseline.locale,
+    baseline.normalized_os AS os,
+    baseline.normalized_os_version AS os_version,
+    metrics.is_default_browser,
+    is_daily_user,
+    is_weekly_user,
+    is_monthly_user,
+    is_dau,
+    is_wau,
+    is_mau
+  FROM
+    baseline
+  LEFT JOIN
+    metrics
+    ON baseline.client_id = metrics.client_id
+    AND baseline.normalized_channel IS NOT DISTINCT FROM metrics.normalized_channel
+)
+SELECT
+  unioned.* EXCEPT (
+    client_id,
+    is_daily_user,
+    is_weekly_user,
+    is_monthly_user,
+    is_dau,
+    is_wau,
+    is_mau
+  ),
+  COUNTIF(is_daily_user) AS daily_users,
+  COUNTIF(is_weekly_user) AS weekly_users,
+  COUNTIF(is_monthly_user) AS monthly_users,
+  COUNTIF(is_dau) AS dau,
+  COUNTIF(is_wau) AS wau,
+  COUNTIF(is_mau) AS mau
+FROM
+  unioned
+GROUP BY
+  submission_date,
+  app_name,
+  app_version,
+  channel,
+  country,
+  city,
+  locale,
+  os,
+  os_version,
+  is_default_browser

sql/moz-fx-data-shared-prod/fenix_derived/locale_aggregates_v1/schema.yaml

@@ -0,0 +1,65 @@
+fields:
+- name: submission_date
+  type: DATE
+  mode: NULLABLE
+  description: Date when ping is received on the server side.
+- name: app_name
+  type: STRING
+  mode: NULLABLE
+  description: Browser name.
+- name: app_version
+  type: STRING
+  mode: NULLABLE
+  description: Browser version installed on the client.
+- name: channel
+  type: STRING
+  mode: NULLABLE
+  description: Browser installation channel installed on the client.
+- name: country
+  type: STRING
+  mode: NULLABLE
+  description: Country reported by the client.
+- name: city
+  type: STRING
+  mode: NULLABLE
+  description: City reported by the client.
+- name: locale
+  type: STRING
+  mode: NULLABLE
+  description: Locale reported by the client, which is a combination of language and regional settings.
+- name: os
+  type: STRING
+  mode: NULLABLE
+  description: Operating System reported by the client.
+- name: os_version
+  type: STRING
+  mode: NULLABLE
+  description: OS version with windows_build_number as reported by the client.
+- name: is_default_browser
+  type: BOOLEAN
+  mode: NULLABLE
+  description: Whether the browser is set as the default browser on the client side.
+- name: daily_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Count of users who report a ping in a day.
+- name: weekly_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Count of users who have reported a ping over the last 7 days.
+- name: monthly_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Count of users who have reported a ping over the last 28 days.
+- name: dau
+  type: INTEGER
+  mode: NULLABLE
+  description: Count of users who reported a ping on the submission_date that qualify as active.
+- name: wau
+  type: INTEGER
+  mode: NULLABLE
+  description: Count of users who have reported a ping over the last 7 days and qualify as active.
+- name: mau
+  type: INTEGER
+  mode: NULLABLE
+  description: Count of users who have reported a ping over the last 28 days and qualify as active.

sql/moz-fx-data-shared-prod/fenix_derived/new_profile_activation_v1/query.sql

@@ -35,9 +35,9 @@ client_search AS (
     USING (client_id)
   WHERE
     (submission_date BETWEEN DATE_SUB(@submission_date, INTERVAL 3 DAY) AND @submission_date)
-    AND normalized_app_name = 'Fenix'
+    AND normalized_app_name_os = "Firefox Android"
   GROUP BY
-    1
+    client_id
 ),
 dou AS (
   SELECT
@@ -71,7 +71,7 @@ adjust_client AS (
     AND metrics.string.first_session_network IS NOT NULL
     AND metrics.string.first_session_network <> ''
   GROUP BY
-    1
+    client_id
 )
 SELECT
   client_id,
@@ -93,7 +93,7 @@ SELECT
   first_seen_date,
   submission_date,
   1 AS new_profile,
-  CAST(days_2_7 > 1 AND COALESCE(search_count, 0) > 0 AS integer) AS activated
+  CAST(days_2_7 > 1 AND COALESCE(search_count, 0) > 0 AS INTEGER) AS activated
 FROM
   dou
 INNER JOIN

sql/moz-fx-data-shared-prod/firefox_accounts/funnel_events_source/view.sql

@@ -1,7 +0,0 @@
-CREATE OR REPLACE VIEW
-  `moz-fx-data-shared-prod.firefox_accounts.funnel_events_source`
-AS
-SELECT
-  *
-FROM
-  `moz-fx-data-shared-prod.firefox_accounts_derived.funnel_events_source_v1`

sql/moz-fx-data-shared-prod/firefox_accounts_derived/funnel_events_source_v1/metadata.yaml

@@ -1,21 +0,0 @@
-friendly_name: Funnel Events Source
-description: |-
-  The materialized events used to create the events_daily rollup.
-  Not exposed as a view since it's not meant for consumption;
-  use firefox_accounts.fxa_all_events
-  instead. This is materialized solely because
-  the events_daily queries are too complex for BigQuery
-  otherwise.
-owners:
-- wlachance@mozilla.com
-labels:
-  table_type: client_level
-scheduling:
-  dag_name: bqetl_event_rollup
-  arguments: ['--schema_update_option=ALLOW_FIELD_ADDITION']
-  task_name: funnel_events_source__v1
-bigquery:
-  time_partitioning:
-    type: day
-    field: submission_date
-    require_partition_filter: false

sql/moz-fx-data-shared-prod/firefox_accounts_derived/funnel_events_source_v1/query.sql

@@ -1,58 +0,0 @@
-SELECT
-  DATE(`timestamp`) AS submission_date,
-  user_id AS client_id,
-  `moz-fx-data-shared-prod`.udf.safe_sample_id(user_id) AS sample_id,
-  SPLIT(event_type, ' - ')[OFFSET(0)] AS category,
-  SPLIT(event_type, ' - ')[OFFSET(1)] AS event,
-  [
-    STRUCT('service' AS key, service AS value),
-    STRUCT('email_type' AS key, email_type AS value),
-    STRUCT('oauth_client_id' AS key, oauth_client_id AS value),
-    STRUCT('connect_device_flow' AS key, connect_device_flow AS value),
-    STRUCT('connect_device_os' AS key, connect_device_os AS value),
-    STRUCT('sync_device_count' AS key, sync_device_count AS value),
-    STRUCT('email_sender' AS key, email_sender AS value),
-    STRUCT('email_service' AS key, email_service AS value),
-    STRUCT('email_template' AS key, email_template AS value),
-    STRUCT('email_version' AS key, email_version AS value)
-  ] AS extra,
-  CAST([] AS ARRAY<STRUCT<key STRING, value STRING>>) AS experiments,
-  logger,
-  event_type,
-  app_version,
-  os_name,
-  os_version,
-  country,
-  `language`,
-  user_id,
-  device_id,
-  `timestamp`,
-  receiveTimestamp,
-  utm_term,
-  utm_source,
-  utm_medium,
-  utm_campaign,
-  utm_content,
-  ua_version,
-  ua_browser,
-  entrypoint,
-  flow_id,
-  service,
-  email_type,
-  email_provider,
-  oauth_client_id,
-  connect_device_flow,
-  connect_device_os,
-  sync_device_count,
-  sync_active_devices_day,
-  sync_active_devices_week,
-  sync_active_devices_month,
-  email_sender,
-  email_service,
-  email_template,
-  email_version,
-FROM
-  `moz-fx-data-shared-prod.firefox_accounts.fxa_all_events`
-WHERE
-  DATE(`timestamp`) = @submission_date
-  AND fxa_log IN ('content', 'auth', 'oauth')

sql/moz-fx-data-shared-prod/firefox_accounts_derived/funnel_events_source_v1/schema.yaml

@@ -1,138 +0,0 @@
-fields:
-- name: submission_date
-  type: DATE
-  mode: NULLABLE
-- name: client_id
-  type: STRING
-  mode: NULLABLE
-- name: sample_id
-  type: INTEGER
-  mode: NULLABLE
-- name: category
-  type: STRING
-  mode: NULLABLE
-- name: event
-  type: STRING
-  mode: NULLABLE
-- name: extra
-  type: RECORD
-  mode: REPEATED
-  fields:
-  - name: key
-    type: STRING
-    mode: NULLABLE
-  - name: value
-    type: STRING
-    mode: NULLABLE
-- name: experiments
-  type: RECORD
-  mode: REPEATED
-  fields:
-  - name: key
-    type: STRING
-    mode: NULLABLE
-  - name: value
-    type: STRING
-    mode: NULLABLE
-- name: logger
-  type: STRING
-  mode: NULLABLE
-- name: event_type
-  type: STRING
-  mode: NULLABLE
-- name: app_version
-  type: STRING
-  mode: NULLABLE
-- name: os_name
-  type: STRING
-  mode: NULLABLE
-- name: os_version
-  type: STRING
-  mode: NULLABLE
-- name: country
-  type: STRING
-  mode: NULLABLE
-- name: language
-  type: STRING
-  mode: NULLABLE
-- name: user_id
-  type: STRING
-  mode: NULLABLE
-- name: device_id
-  type: STRING
-  mode: NULLABLE
-- name: timestamp
-  type: TIMESTAMP
-  mode: NULLABLE
-- name: receiveTimestamp
-  type: TIMESTAMP
-  mode: NULLABLE
-- name: utm_term
-  type: STRING
-  mode: NULLABLE
-- name: utm_source
-  type: STRING
-  mode: NULLABLE
-- name: utm_medium
-  type: STRING
-  mode: NULLABLE
-- name: utm_campaign
-  type: STRING
-  mode: NULLABLE
-- name: utm_content
-  type: STRING
-  mode: NULLABLE
-- name: ua_version
-  type: STRING
-  mode: NULLABLE
-- name: ua_browser
-  type: STRING
-  mode: NULLABLE
-- name: entrypoint
-  type: STRING
-  mode: NULLABLE
-- name: flow_id
-  type: STRING
-  mode: NULLABLE
-- name: service
-  type: STRING
-  mode: NULLABLE
-- name: email_type
-  type: STRING
-  mode: NULLABLE
-- name: email_provider
-  type: STRING
-  mode: NULLABLE
-- name: oauth_client_id
-  type: STRING
-  mode: NULLABLE
-- name: connect_device_flow
-  type: STRING
-  mode: NULLABLE
-- name: connect_device_os
-  type: STRING
-  mode: NULLABLE
-- name: sync_device_count
-  type: STRING
-  mode: NULLABLE
-- name: sync_active_devices_day
-  type: STRING
-  mode: NULLABLE
-- name: sync_active_devices_week
-  type: STRING
-  mode: NULLABLE
-- name: sync_active_devices_month
-  type: STRING
-  mode: NULLABLE
-- name: email_sender
-  type: STRING
-  mode: NULLABLE
-- name: email_service
-  type: STRING
-  mode: NULLABLE
-- name: email_template
-  type: STRING
-  mode: NULLABLE
-- name: email_version
-  type: STRING
-  mode: NULLABLE

sql/moz-fx-data-shared-prod/firefox_accounts_derived/health_counts_v1/metadata.yaml

@@ -0,0 +1,12 @@
+friendly_name: Accounts Health Counts
+description: >
+  Counts calculated daily regarding the health of the Accounts service.  E.g. # of issues open.
+owners:
+  - wclouser@mozilla.com
+external_data:
+  format: google_sheets
+  source_uris:
+    - https://docs.google.com/spreadsheets/d/1uGHOS-cY_OUqicbBkS2Z20EK0hLq3PDw28TZcxP5RHI/edit
+  options:
+    skip_leading_rows: 1
+    range: Counts

sql/moz-fx-data-shared-prod/firefox_accounts_derived/health_counts_v1/schema.yaml

@@ -0,0 +1,13 @@
+fields:
+  - mode: NULLABLE
+    name: date
+    type: TIMESTAMP
+  - mode: NULLABLE
+    name: type
+    type: STRING
+  - mode: NULLABLE
+    name: state
+    type: STRING
+  - mode: NULLABLE
+    name: count
+    type: INTEGER

sql/moz-fx-data-shared-prod/firefox_accounts_derived/health_issues_v1/metadata.yaml

@@ -0,0 +1,12 @@
+friendly_name: Accounts Health Issues
+description: >
+  Mirrors Jira Issue data for Firefox Accounts.
+owners:
+  - wclouser@mozilla.com
+external_data:
+  format: google_sheets
+  source_uris:
+    - https://docs.google.com/spreadsheets/d/1uGHOS-cY_OUqicbBkS2Z20EK0hLq3PDw28TZcxP5RHI/edit
+  options:
+    skip_leading_rows: 1
+    range: Issue Data

sql/moz-fx-data-shared-prod/firefox_accounts_derived/health_issues_v1/schema.yaml

@@ -0,0 +1,61 @@
+fields:
+  - mode: NULLABLE
+    name: key
+    type: STRING
+  - mode: NULLABLE
+    name: permalink
+    type: STRING
+  - mode: NULLABLE
+    name: id
+    type: INTEGER
+  - mode: NULLABLE
+    name: assignee
+    type: STRING
+  - mode: NULLABLE
+    name: created
+    type: TIMESTAMP
+  - mode: NULLABLE
+    name: found_in
+    type: STRING
+  - mode: NULLABLE
+    name: issuetype
+    type: STRING
+  - mode: NULLABLE
+    name: reporter
+    type: STRING
+  - mode: NULLABLE
+    name: resolution
+    type: STRING
+  - mode: NULLABLE
+    name: resolutiondate
+    type: TIMESTAMP
+  - mode: NULLABLE
+    name: status
+    type: STRING
+  - mode: NULLABLE
+    name: storypoints
+    type: INTEGER
+  - mode: NULLABLE
+    name: summary
+    type: STRING
+  - mode: NULLABLE
+    name: updated
+    type: TIMESTAMP
+  - mode: NULLABLE
+    name: epic
+    type: STRING
+  - mode: NULLABLE
+    name: originalstorypoints
+    type: STRING
+  - mode: NULLABLE
+    name: timetoresolve
+    type: INTEGER
+  - mode: NULLABLE
+    name: severity
+    type: STRING
+  - mode: NULLABLE
+    name: priority
+    type: STRING
+  - mode: NULLABLE
+    name: labels
+    type: STRING

sql/moz-fx-data-shared-prod/firefox_accounts_derived/health_lighthouse_v1/metadata.yaml

@@ -0,0 +1,12 @@
+friendly_name: Accounts Health Lighthouse
+description: >
+  Front-end health metrics for Firefox Accounts, calculated from Lighthouse reports.
+owners:
+  - wclouser@mozilla.com
+external_data:
+  format: google_sheets
+  source_uris:
+    - https://docs.google.com/spreadsheets/d/1uGHOS-cY_OUqicbBkS2Z20EK0hLq3PDw28TZcxP5RHI/edit
+  options:
+    skip_leading_rows: 1
+    range: Lighthouse

sql/moz-fx-data-shared-prod/firefox_accounts_derived/health_lighthouse_v1/schema.yaml

@@ -0,0 +1,22 @@
+fields:
+  - mode: NULLABLE
+    name: fetchTime
+    type: TIMESTAMP
+  - mode: NULLABLE
+    name: url
+    type: STRING
+  - mode: NULLABLE
+    name: category
+    type: STRING
+  - mode: NULLABLE
+    name: audit
+    type: STRING
+  - mode: NULLABLE
+    name: score
+    type: NUMERIC
+  - mode: NULLABLE
+    name: numVal
+    type: NUMERIC
+  - mode: NULLABLE
+    name: formfactor
+    type: STRING

sql/moz-fx-data-shared-prod/firefox_accounts_derived/health_pingdom_v1/metadata.yaml

@@ -0,0 +1,12 @@
+friendly_name: Accounts Health Pingdom
+description: >
+  Records pingdom values for the Accounts service.
+owners:
+  - wclouser@mozilla.com
+external_data:
+  format: google_sheets
+  source_uris:
+    - https://docs.google.com/spreadsheets/d/1uGHOS-cY_OUqicbBkS2Z20EK0hLq3PDw28TZcxP5RHI/edit
+  options:
+    skip_leading_rows: 1
+    range: Pingdom Data

sql/moz-fx-data-shared-prod/firefox_accounts_derived/health_pingdom_v1/schema.yaml

@@ -0,0 +1,25 @@
+fields:
+  - mode: NULLABLE
+    name: avgresponse
+    type: INTEGER
+  - mode: NULLABLE
+    name: downtime
+    type: INTEGER
+  - mode: NULLABLE
+    name: starttime
+    type: TIMESTAMP
+  - mode: NULLABLE
+    name: unmonitored
+    type: INTEGER
+  - mode: NULLABLE
+    name: uptime
+    type: INTEGER
+  - mode: NULLABLE
+    name: project
+    type: STRING
+  - mode: NULLABLE
+    name: fuzzy_uptime_percent
+    type: FLOAT
+  - mode: NULLABLE
+    name: unmonitored_percent
+    type: FLOAT

sql/moz-fx-data-shared-prod/firefox_accounts_derived/health_prs_v1/metadata.yaml

@@ -0,0 +1,12 @@
+friendly_name: Accounts Health PRs
+description: >
+  Mirrors data about Account's GitHub PRs.
+owners:
+  - wclouser@mozilla.com
+external_data:
+  format: google_sheets
+  source_uris:
+    - https://docs.google.com/spreadsheets/d/1uGHOS-cY_OUqicbBkS2Z20EK0hLq3PDw28TZcxP5RHI/edit
+  options:
+    skip_leading_rows: 1
+    range: Pull Request Data

sql/moz-fx-data-shared-prod/firefox_accounts_derived/health_prs_v1/schema.yaml

@@ -0,0 +1,46 @@
+fields:
+  - mode: NULLABLE
+    name: id
+    type: INTEGER
+  - mode: NULLABLE
+    name: number
+    type: INTEGER
+  - mode: NULLABLE
+    name: url
+    type: STRING
+  - mode: NULLABLE
+    name: title
+    type: STRING
+  - mode: NULLABLE
+    name: state
+    type: STRING
+  - mode: NULLABLE
+    name: author
+    type: STRING
+  - mode: NULLABLE
+    name: created_at
+    type: TIMESTAMP
+  - mode: NULLABLE
+    name: updated_at
+    type: TIMESTAMP
+  - mode: NULLABLE
+    name: closed_at
+    type: TIMESTAMP
+  - mode: NULLABLE
+    name: merged_at
+    type: TIMESTAMP
+  - mode: NULLABLE
+    name: draft
+    type: BOOLEAN
+  - mode: NULLABLE
+    name: time_to_close
+    type: INTEGER
+  - mode: NULLABLE
+    name: reviewers
+    type: STRING
+  - mode: NULLABLE
+    name: has_an_issue
+    type: BOOLEAN
+  - mode: NULLABLE
+    name: time_to_merge
+    type: INTEGER

sql/moz-fx-data-shared-prod/firefox_accounts_derived/health_tags_v1/metadata.yaml

@@ -0,0 +1,12 @@
+friendly_name: Accounts Health Tags
+description: >
+  Tracks push timings for Firefox Accounts.
+owners:
+  - wclouser@mozilla.com
+external_data:
+  format: google_sheets
+  source_uris:
+    - https://docs.google.com/spreadsheets/d/1uGHOS-cY_OUqicbBkS2Z20EK0hLq3PDw28TZcxP5RHI/edit
+  options:
+    skip_leading_rows: 1
+    range: Tag Data

sql/moz-fx-data-shared-prod/firefox_accounts_derived/health_tags_v1/schema.yaml

@@ -0,0 +1,22 @@
+fields:
+  - mode: NULLABLE
+    name: timestamp
+    type: TIMESTAMP
+  - mode: NULLABLE
+    name: server
+    type: STRING
+  - mode: NULLABLE
+    name: version
+    type: STRING
+  - mode: NULLABLE
+    name: commit
+    type: STRING
+  - mode: NULLABLE
+    name: l10n
+    type: STRING
+  - mode: NULLABLE
+    name: tospp
+    type: STRING
+  - mode: NULLABLE
+    name: since_last_push
+    type: FLOAT

sql/moz-fx-data-shared-prod/firefox_desktop/locale_aggregates/view.sql

@@ -0,0 +1,7 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.firefox_desktop.locale_aggregates`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod.firefox_desktop_derived.locale_aggregates_v1`

sql/moz-fx-data-shared-prod/firefox_desktop_background_defaultagent/dataset_metadata.yaml

@@ -0,0 +1,16 @@
+friendly_name: Firefox Desktop Background Defaultagent
+description: |-
+  Data related to firefox desktop background default agent
+dataset_base_acl: view
+user_facing: true
+labels: {}
+default_table_workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential
+default_table_expiration_ms: null
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential
+syndication: {}

sql/moz-fx-data-shared-prod/firefox_desktop_background_defaultagent/default_agent_agg/view.sql

@@ -0,0 +1,7 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.firefox_desktop_background_defaultagent.default_agent_agg`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod.firefox_desktop_background_defaultagent_derived.default_agent_agg_v1`

sql/moz-fx-data-shared-prod/firefox_desktop_background_defaultagent_derived/dataset_metadata.yaml

@@ -0,0 +1,16 @@
+friendly_name: Firefox Desktop Background Defaultagent Derived
+description: |-
+  Data related to firefox desktop background default agent
+dataset_base_acl: derived
+user_facing: false
+labels: {}
+default_table_workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential
+default_table_expiration_ms: null
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential
+syndication: {}

sql/moz-fx-data-shared-prod/firefox_desktop_background_defaultagent_derived/default_agent_agg_v1/backfill.yaml

@@ -0,0 +1,9 @@
+2024-12-27:
+  start_date: 2023-01-01
+  end_date: 2023-12-31
+  reason: https://mozilla-hub.atlassian.net/browse/DENG-6889
+  watchers:
+  - wichan@mozilla.com
+  status: Complete
+  shredder_mitigation: false
+  override_retention_limit: false

sql/moz-fx-data-shared-prod/firefox_desktop_background_defaultagent_derived/default_agent_agg_v1/metadata.yaml

@@ -0,0 +1,20 @@
+friendly_name: Firefox Desktop Background Default Agent Derived
+description: |-
+  Aggregate table for firefox desktop background default agent
+owners:
+- wichan@mozilla.com
+labels:
+  incremental: true
+  owner1: wichan@mozilla.com
+  table_type: aggregate
+  shredder_mitigation: true
+scheduling:
+  dag_name: bqetl_default_browser_aggregates
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false
+    expiration_days: null
+  range_partitioning: null
+references: {}

sql/moz-fx-data-shared-prod/firefox_desktop_background_defaultagent_derived/default_agent_agg_v1/query.sql

@@ -0,0 +1,21 @@
+SELECT
+  DATE(submission_timestamp) AS submission_date,
+  mozfun.norm.truncate_version(client_info.app_display_version, 'major') AS build_version_major,
+  normalized_channel,
+  metrics.string.system_default_browser,
+  metrics.string.system_default_previous_browser,
+  metrics.boolean.notification_show_success,
+  metrics.string.notification_action,
+  COUNT(*) AS row_count
+FROM
+  `moz-fx-data-shared-prod.firefox_desktop_background_defaultagent.default_agent`
+WHERE
+  DATE(submission_timestamp) = @submission_date
+GROUP BY
+  DATE(submission_timestamp),
+  mozfun.norm.truncate_version(client_info.app_display_version, 'major'),
+  normalized_channel,
+  metrics.string.system_default_browser,
+  metrics.string.system_default_previous_browser,
+  metrics.boolean.notification_show_success,
+  metrics.string.notification_action

sql/moz-fx-data-shared-prod/firefox_desktop_background_defaultagent_derived/default_agent_agg_v1/schema.yaml

@@ -0,0 +1,33 @@
+fields:
+- name: submission_date
+  type: DATE
+  mode: NULLABLE
+  description: Date when the ingestion edge server accepted this message.
+- name: build_version_major
+  type: NUMERIC
+  mode: NULLABLE
+  description: Major build version.
+- name: normalized_channel
+  type: STRING
+  mode: NULLABLE
+  description: normalized channel.
+- name: system_default_browser
+  type: STRING
+  mode: NULLABLE
+  description: Which browser is currently set as the system default web browser.
+- name: system_default_previous_browser
+  type: STRING
+  mode: NULLABLE
+  description: Which browser was set as the system default before it was changed to the current setting.
+- name: notification_show_success
+  type: BOOLEAN
+  mode: NULLABLE
+  description: Whether a notification was shown or not. Possible value include "shown" and "error".
+- name: notification_action
+  type: STRING
+  mode: NULLABLE
+  description: The action that the user took in response to the notification.
+- name: row_count
+  type: INTEGER
+  mode: NULLABLE
+  description: The total number of rows aggregated.

sql/moz-fx-data-shared-prod/firefox_desktop_derived/desktop_installs_v1/metadata.yaml

@@ -6,6 +6,8 @@ owners:
 labels:
   incremental: true
   owner1: example
+  table_type: aggregate
+  shredder_mitigation: true
 scheduling:
   dag_name: bqetl_desktop_installs_v1
 bigquery:

sql/moz-fx-data-shared-prod/firefox_desktop_derived/locale_aggregates_v1/metadata.yaml

@@ -0,0 +1,23 @@
+friendly_name: Locale Aggregates
+description: |-
+  Firefox Desktop DAU, WAU, MAU and client counts by locale and core dimensions of analysis.
+owners:
+- lvargas@mozilla.com
+labels:
+  incremental: true
+  shredder_mitigation: true
+scheduling:
+  dag_name: bqetl_analytics_aggregations
+  date_partition_offset: -1
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: true
+    expiration_days: null
+  clustering:
+    fields:
+    - country
+    - channel
+    - locale
+references: {}

sql/moz-fx-data-shared-prod/firefox_desktop_derived/locale_aggregates_v1/query.sql

@@ -0,0 +1,57 @@
+-- Query for firefox_desktop_derived.locale_aggregates_v1
+WITH todays_metrics AS (
+  SELECT
+    client_id,
+    submission_date,
+    app_name,
+    app_version,
+    normalized_channel AS channel,
+    IFNULL(country, '??') AS country,
+    city,
+    locale,
+    os,
+    COALESCE(
+      `mozfun.norm.windows_version_info`(os, normalized_os_version, windows_build_number),
+      normalized_os_version
+    ) AS os_version,
+    is_default_browser,
+    is_daily_user,
+    is_weekly_user,
+    is_monthly_user,
+    is_dau,
+    is_wau,
+    is_mau
+  FROM
+    `moz-fx-data-shared-prod.telemetry.desktop_active_users`
+  WHERE
+    submission_date = @submission_date
+)
+SELECT
+  todays_metrics.* EXCEPT (
+    client_id,
+    is_daily_user,
+    is_weekly_user,
+    is_monthly_user,
+    is_dau,
+    is_wau,
+    is_mau
+  ),
+  COUNTIF(is_daily_user) AS daily_users,
+  COUNTIF(is_weekly_user) AS weekly_users,
+  COUNTIF(is_monthly_user) AS monthly_users,
+  COUNTIF(is_dau) AS dau,
+  COUNTIF(is_wau) AS wau,
+  COUNTIF(is_mau) AS mau
+FROM
+  todays_metrics
+GROUP BY
+  submission_date,
+  app_name,
+  app_version,
+  channel,
+  country,
+  city,
+  locale,
+  os,
+  os_version,
+  is_default_browser

sql/moz-fx-data-shared-prod/firefox_desktop_derived/locale_aggregates_v1/schema.yaml

@@ -0,0 +1,65 @@
+fields:
+- name: submission_date
+  type: DATE
+  mode: NULLABLE
+  description: Date when ping is received on the server side.
+- name: app_name
+  type: STRING
+  mode: NULLABLE
+  description: Browser name.
+- name: app_version
+  type: STRING
+  mode: NULLABLE
+  description: Browser version installed on the client.
+- name: channel
+  type: STRING
+  mode: NULLABLE
+  description: Browser installation channel installed on the client.
+- name: country
+  type: STRING
+  mode: NULLABLE
+  description: Country reported by the client.
+- name: city
+  type: STRING
+  mode: NULLABLE
+  description: City reported by the client.
+- name: locale
+  type: STRING
+  mode: NULLABLE
+  description: Locale reported by the client, which is a combination of language and regional settings.
+- name: os
+  type: STRING
+  mode: NULLABLE
+  description: Operating System reported by the client.
+- name: os_version
+  type: STRING
+  mode: NULLABLE
+  description: OS version with windows_build_number as reported by the client.
+- name: is_default_browser
+  type: BOOLEAN
+  mode: NULLABLE
+  description: Whether the browser is set as the default browser on the client side.
+- name: daily_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Count of users who report a ping in a day.
+- name: weekly_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Count of users who have reported a ping over the last 7 days.
+- name: monthly_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Count of users who have reported a ping over the last 28 days.
+- name: dau
+  type: INTEGER
+  mode: NULLABLE
+  description: Count of users who reported a ping on the submission_date that qualify as active.
+- name: wau
+  type: INTEGER
+  mode: NULLABLE
+  description: Count of users who have reported a ping over the last 7 days and qualify as active.
+- name: mau
+  type: INTEGER
+  mode: NULLABLE
+  description: Count of users who have reported a ping over the last 28 days and qualify as active.

sql/moz-fx-data-shared-prod/firefox_ios/app_store_choice_screen_selection/view.sql

@@ -0,0 +1,19 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.firefox_ios.app_store_choice_screen_selection`
+AS
+SELECT
+  * EXCEPT (logical_date) REPLACE(DATE(`date`) AS `date`),
+FROM
+  `moz-fx-data-shared-prod.firefox_ios_derived.app_store_choice_screen_selection_v1`
+QUALIFY
+  ROW_NUMBER() OVER (
+    PARTITION BY
+      `date`,
+      territory,
+      build,
+      device,
+      platform,
+      release_type
+    ORDER BY
+      logical_date DESC
+  ) = 1

sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_choice_screen_selection_v1/metadata.yaml

@@ -0,0 +1,41 @@
+friendly_name: Apple App Store Choice Screen Selection (post-iOS 18.2 only)
+description: |
+  Apple App Store Choice Screen Engagement (post-iOS 18.2 only)
+  This report includes rates around user selection/egagement with our browser
+  via the App Store Choice Screen.
+
+  Official docs: https://developer.apple.com/documentation/analytics-reports/browser-choice-screen-selection
+
+owners:
+- kik@mozilla.com
+labels:
+  schedule: daily
+  incremental: true
+  table_type: aggregate
+  shredder_mitigation: false
+scheduling:
+  dag_name: bqetl_firefox_ios
+  depends_on_past: false
+  arguments:
+  - --date={{macros.ds_add(ds, -5)}}
+  - --connect_app_id=989804926
+  - --partition_field=logical_date
+
+  secrets:
+  - deploy_target: CONNECT_ISSUER_ID
+    key: bqetl_firefox_ios__app_store_connect_issuer_id
+  - deploy_target: CONNECT_KEY_ID
+    key: bqetl_firefox_ios__app_store_connect_key_id
+  - deploy_target: CONNECT_KEY
+    key: bqetl_firefox_ios__app_store_connect_key
+
+  date_partition_offset: -5
+  retry_delay: 30m
+  retries: 2
+  email_on_retry: false
+
+bigquery:
+  time_partitioning:
+    type: day
+    field: logical_date
+    require_partition_filter: false

sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_choice_screen_selection_v1/query.py

@@ -0,0 +1,260 @@
+"""Fetch Analytics Report for Choice Screen Engagement in the App store via App Store Connect."""
+
+import gzip
+import io
+import logging
+import os
+import tempfile
+import time
+from argparse import ArgumentParser
+from pathlib import Path
+from sys import exit
+
+import jwt
+import pandas as pd
+import requests
+from google.cloud import bigquery
+
+from bigquery_etl.schema import Schema
+
+#
+# https://developer.apple.com/documentation/AppStoreConnectAPI/downloading-analytics-reports
+CONNECT_ISSUER_ID = os.environ.get("CONNECT_ISSUER_ID")
+CONNECT_KEY_ID = os.environ.get("CONNECT_KEY_ID")
+CONNECT_KEY = os.environ.get("CONNECT_KEY")
+
+HOST = "https://api.appstoreconnect.apple.com"
+REPORT_TITLE = "Browser Choice Screen Selection"
+REPORT_DATA_DELIMITER = ","
+
+SCHEMA_FILE = Path(__file__).parent / "schema.yaml"
+SCHEMA = Schema.from_schema_file(SCHEMA_FILE).to_bigquery_schema()
+
+
+def generate_jwt_token(issuer_id, key_id, private_key):
+    """Generate jtw token to be used for API authentication."""
+    payload = {
+        "iss": issuer_id,
+        "iat": int(time.time()),
+        "exp": int(time.time()) + 20 * 60,  # Token valid for 20 minutes
+        "aud": "appstoreconnect-v1",
+    }
+    algorithm = "ES256"
+
+    return jwt.encode(
+        payload,
+        private_key.replace("\\n", "\n"),
+        algorithm=algorithm,
+        headers={"kid": key_id, "alg": algorithm, "typ": "JWT"},
+    )
+
+
+def download_gz_file(url, target_file_path):
+    """Download and decompress a gz file."""
+    with requests.get(url, stream=True) as response:
+        response.raise_for_status()
+        file_content = io.BytesIO()
+
+        for chunk in response.iter_content(chunk_size=8192):
+            if chunk:  # Filter out keep-alive new chunks
+                file_content.write(chunk)
+
+        # # Move the file content pointer to the beginning for reading
+        file_content.seek(0)
+
+        # # Decompress from in-memory buffer to file
+        with gzip.GzipFile(fileobj=file_content, mode="rb") as f_in:
+            with open(target_file_path, "wb") as f_out:
+                f_out.write(f_in.read())
+
+
+def api_call(url, jwt_token):
+    """Handle making API requests to App Store Connect."""
+    headers = {
+        "Authorization": f"Bearer {jwt_token}",
+        "Content-Type": "application/json",
+    }
+
+    response = requests.get(url, headers=headers)
+    response.raise_for_status()
+
+    return response.json()
+
+
+def get_paginated_data(url, jwt_token):
+    """Get data from all result pages."""
+    data = []
+    while True:
+        result = api_call(url, jwt_token)
+        data.extend(result["data"])
+        if next := result["links"].get("next"):
+            url = next
+        else:
+            return data
+
+
+def fetch_app_reports(jwt_token, app_id):
+    """Retrieve the report requests for the given app."""
+    endpoint = f"apps/{app_id}/analyticsReportRequests"
+    response_data = api_call(f"{HOST}/v1/{endpoint}", jwt_token)
+    return response_data.get("data")
+
+
+def fetch_report_instances(jwt_token, report_id):
+    """Retrieve a list of report instances from the App Store."""
+    endpoint = f"v1/analyticsReportInstances/{report_id}/segments"
+    response_data = api_call(f"{HOST}/{endpoint}", jwt_token)
+    return response_data.get("data")
+
+
+def fetch_report_data(app_id, date, jwt_token, target_file_path):
+    """Fetch and prints the report data."""
+    # "ONGOING" provides current data and generates reports daily, weekly and monthly.
+    # assumed there is only 1 ongoing report request at any given time
+    ongoing_analytics_report_request = list(
+        filter(
+            lambda x: x["attributes"]["accessType"] == "ONGOING",
+            fetch_app_reports(jwt_token, app_id),
+        )
+    )[0]
+
+    analytics_reports = get_paginated_data(
+        ongoing_analytics_report_request["relationships"]["reports"]["links"][
+            "related"
+        ],
+        jwt_token,
+    )
+
+    # for now we just assume there is only one ongoing browser choice screen report
+    browser_choice_screen_analytics_report = list(
+        filter(
+            lambda x: x["attributes"]["name"] == REPORT_TITLE,
+            analytics_reports,
+        )
+    )[0]
+
+    report_instances = api_call(
+        browser_choice_screen_analytics_report["relationships"]["instances"]["links"][
+            "related"
+        ],
+        jwt_token,
+    )["data"]
+
+    daily_report_instances = list(
+        filter(lambda x: x["attributes"]["granularity"] == "DAILY", report_instances)
+    )
+
+    try:
+        specific_date_report_instance = list(
+            filter(
+                lambda x: x["attributes"]["processingDate"] == date,
+                daily_report_instances,
+            )
+        )[0]
+    except IndexError:
+        logging.error(
+            f"It appears that report: `{REPORT_TITLE}` for date: `{date}` does not exist, potentially it has not yet been made available."
+        )
+        exit(1)
+
+    analytics_report_segments = api_call(
+        specific_date_report_instance["relationships"]["segments"]["links"]["related"],
+        jwt_token,
+    )["data"][0]
+
+    checksum = analytics_report_segments["attributes"]["checksum"]
+
+    response = requests.get(analytics_report_segments["attributes"]["url"])
+    response.raise_for_status()
+
+    download_gz_file(
+        analytics_report_segments["attributes"]["url"],
+        target_file_path=target_file_path,
+    )
+
+    return target_file_path, checksum
+
+
+def upload_to_bigquery(
+    local_file_path, project, dataset, table_name, date, partition_field
+):
+    """Upload the data to bigquery."""
+    df = pd.read_csv(local_file_path, delimiter=REPORT_DATA_DELIMITER)
+    df[partition_field] = date
+    df.columns = [x.lower().replace(" ", "_") for x in df.columns]
+
+    df["date"] = pd.to_datetime(df["date"], format="%Y-%m-%d")
+    df[partition_field] = pd.to_datetime(df["logical_date"], format="%Y-%m-%d")
+
+    job_config = bigquery.LoadJobConfig(
+        create_disposition=bigquery.CreateDisposition.CREATE_IF_NEEDED,
+        write_disposition=bigquery.WriteDisposition.WRITE_TRUNCATE,
+        schema=SCHEMA,
+    )
+
+    client = bigquery.Client(project)
+    destination = f"{project}.{dataset}.{table_name}${date.replace('-', '')}"
+
+    job = client.load_table_from_dataframe(df, destination, job_config=job_config)
+
+    while client.get_job(job.job_id, location=job.location).state != "DONE":
+        logging.info("Waiting for the bq load job to be done, job_id: %s." % job.job_id)
+        time.sleep(5)
+
+    return job.result(), destination
+
+
+def main():
+    """Input data, call functions, get stuff done."""
+    parser = ArgumentParser(description=__doc__)
+    parser.add_argument("--date", required=True)
+    parser.add_argument(
+        "--connect_app_id", required=True, help="App Store Connect App ID"
+    )
+    parser.add_argument("--project", default="moz-fx-data-shared-prod")
+    parser.add_argument("--dataset", default="firefox_ios_derived")
+    parser.add_argument("--table", default="app_store_choice_screen_selection_v1")
+    parser.add_argument("--partition_field", default="logical_date")
+    parser.add_argument(
+        "--connect_issuer_id",
+        default=CONNECT_ISSUER_ID,
+        help="App Store Connect Issuer ID",
+    )
+    parser.add_argument(
+        "--connect_key_id", default=CONNECT_KEY_ID, help="App Store Connect Key ID"
+    )
+    parser.add_argument(
+        "--connect_private_key",
+        default=CONNECT_KEY,
+        help="Private key",
+    )
+    args = parser.parse_args()
+
+    jwt_token = generate_jwt_token(
+        args.connect_issuer_id, args.connect_key_id, args.connect_private_key
+    )
+
+    with tempfile.NamedTemporaryFile() as temp_file:
+        report_file, _ = fetch_report_data(
+            app_id=args.connect_app_id,
+            date=args.date,
+            jwt_token=jwt_token,
+            target_file_path=temp_file.name,
+        )
+
+        logging.info("Report file downloaded: %s" % report_file)
+
+        _, destination_table = upload_to_bigquery(
+            temp_file.name,
+            args.project,
+            args.dataset,
+            args.table,
+            args.date,
+            args.partition_field,
+        )
+
+        logging.info("BigQuery table has been created: %s" % destination_table)
+
+
+if __name__ == "__main__":
+    main()

sql/moz-fx-data-shared-prod/firefox_ios_derived/app_store_choice_screen_selection_v1/schema.yaml

@@ -0,0 +1,92 @@
+fields:
+
+- mode: NULLABLE
+  name: date
+  type: DATETIME
+  description: |
+    Date on which the event occurred. For weekly and monthly instances,
+    this column represents the first day of the week and month, respectively.
+
+- mode: NULLABLE
+  name: territory
+  type: STRING
+  description: |
+    The user’s region code as set in Settings > General > Language & Region. \
+    This may not correspond to the user’s Apple Account or App Store storefront.
+
+- mode: NULLABLE
+  name: build
+  type: STRING
+  description: |
+    Build of device on which event occurred.
+
+- mode: NULLABLE
+  name: device
+  type: STRING
+  description: |
+    The device on which the event occurred.
+
+- mode: NULLABLE
+  name: platform
+  type: STRING
+  description: |
+    OS version on the device on which the event occurred
+
+- mode: NULLABLE
+  name: release_type
+  type: STRING
+  description: |
+    Type of software release.
+
+- mode: NULLABLE
+  name: selection_rate
+  type: FLOAT
+  description: |
+    Percentage of unique devices that set your browser app \
+    as default among all unique devices that chose a default.
+
+- mode: NULLABLE
+  name: new_install_rate
+  type: FLOAT
+  description: |
+    Percentage of unique devices that set your browser app \
+    as default without having it installed already, among \
+    all unique devices that chose your browser app as a default.
+
+- mode: NULLABLE
+  name: existing_install_rate
+  type: FLOAT
+  description: |
+    Percentage of unique devices that set your browser app \
+    as default and already had it installed, among \
+    all unique devices that chose your browser app as a default.
+
+- mode: NULLABLE
+  name: product_page_conversion_rate
+  type: FLOAT
+  description: |
+    Percentage of unique devices that set your browser app as default \
+    after viewing its product page, \
+    among all unique devices that viewed your browser app’s product page.
+
+- mode: NULLABLE
+  name: informed_selection_rate
+  type: FLOAT
+  description: |
+    Percentage of unique devices that set your browser app \
+    as default that had previously viewed its product page, \
+    among all unique devices that chose your browser app as a default.
+
+- mode: NULLABLE
+  name: direct_selection_rate
+  type: FLOAT
+  description: |
+    Percentage of unique devices that set your browser app \
+    as default without viewing its product page, \
+    among all unique devices that chose your browser app as a default.
+
+- mode: NULLABLE
+  name: logical_date
+  type: DATETIME
+  description: |
+    Logical date used by the date for processing.

sql/moz-fx-data-shared-prod/firefox_ios_derived/feature_usage_metrics_v2/metadata.yaml

@@ -0,0 +1,20 @@
+friendly_name: Feature Usage Metrics (Firefox iOS)
+description: |-
+  Metrics from metrics pings for mobile feature usage dashboards (iOS)
+owners:
+- rzhao@mozilla.com
+- vsabino@mozilla.com
+labels:
+  application: firefox_ios
+  incremental: true
+  owner1: rzhao
+  table_type: aggregate
+  shredder_mitigation: true
+scheduling:
+  dag_name: bqetl_mobile_feature_usage
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: true
+    expiration_days: null

sql/moz-fx-data-shared-prod/firefox_ios_derived/feature_usage_metrics_v2/query.sql

@@ -0,0 +1,316 @@
+WITH baseline_clients AS (
+  SELECT
+    submission_date AS dau_date,
+    client_id
+  FROM
+    `moz-fx-data-shared-prod.firefox_ios.baseline_clients_daily`
+  WHERE
+    submission_date = DATE_SUB(@submission_date, INTERVAL 4 DAY)
+    AND durations > 0
+    AND LOWER(COALESCE(isp, "")) <> "browserstack"
+),
+client_attribution AS (
+  SELECT
+    client_id,
+    adjust_network,
+  FROM
+    `moz-fx-data-shared-prod.firefox_ios.attribution_clients`
+),
+metrics_dau AS (
+  -- assign a DAU date for each metric ping while keeping it de-duplicated
+  SELECT
+    m.*,
+    MAX(dau_date) OVER (PARTITION BY document_id) AS dau_date
+  FROM
+    `moz-fx-data-shared-prod.firefox_ios.metrics` m
+  JOIN
+    baseline_clients
+    ON client_info.client_id = client_id
+     -- offset by at least one to reflect metrics ping design considerations
+    AND DATE_DIFF(DATE(submission_timestamp), dau_date, DAY)
+    BETWEEN 1
+    AND 4
+  WHERE
+    DATE(submission_timestamp)
+    BETWEEN DATE_SUB(@submission_date, INTERVAL 3 DAY)
+    AND @submission_date
+  QUALIFY
+    ROW_NUMBER() OVER (PARTITION BY document_id ORDER BY submission_timestamp) = 1
+),
+metric_ping_clients_feature_usage AS (
+  SELECT
+    dau_date,
+    client_info.client_id,
+    ARRAY_AGG(normalized_channel ORDER BY submission_timestamp DESC)[SAFE_OFFSET(0)] AS channel,
+    ARRAY_AGG(normalized_country_code ORDER BY submission_timestamp DESC)[
+      SAFE_OFFSET(0)
+    ] AS country,
+    IF(SUM(metrics.counter.app_opened_as_default_browser) > 0, TRUE, FALSE) AS is_default_browser,
+    --Credential Management: Logins
+    SUM(COALESCE(metrics.counter.logins_deleted, 0)) AS logins_deleted,
+    SUM(COALESCE(metrics.counter.logins_modified, 0)) AS logins_modified,
+    SUM(COALESCE(metrics.counter.logins_saved, 0)) AS logins_saved,
+    MAX(COALESCE(metrics.quantity.logins_saved_all, 0)) AS logins_saved_all,
+    --Credential Management: Credit Cards
+    CAST(
+      MAX(COALESCE(metrics.boolean.credit_card_autofill_enabled, FALSE)) AS INT64
+    ) AS credit_card_autofill_enabled,
+    CAST(
+      MAX(COALESCE(metrics.boolean.credit_card_sync_enabled, FALSE)) AS INT64
+    ) AS credit_card_sync_enabled,
+    SUM(COALESCE(metrics.counter.credit_card_deleted, 0)) AS credit_card_deleted,
+    SUM(COALESCE(metrics.counter.credit_card_modified, 0)) AS credit_card_modified,
+    SUM(COALESCE(metrics.counter.credit_card_saved, 0)) AS credit_card_saved,
+    MAX(COALESCE(metrics.quantity.credit_card_saved_all, 0)) AS credit_card_saved_all,
+    --Bookmark
+    SUM(COALESCE(bookmarks_add_table.value, 0)) AS bookmarks_add,
+    SUM(COALESCE(bookmarks_delete_table.value, 0)) AS bookmarks_delete,
+    SUM(COALESCE(bookmarks_edit_table.value, 0)) AS bookmarks_edit,
+    CAST(
+      MAX(COALESCE(metrics.boolean.bookmarks_has_mobile_bookmarks, FALSE)) AS INT64
+    ) AS has_mobile_bookmarks,
+    MAX(COALESCE(metrics.quantity.bookmarks_mobile_bookmarks_count, 0)) AS mobile_bookmarks_count,
+    SUM(COALESCE(bookmarks_open_table.value, 0)) AS bookmarks_open,
+    SUM(COALESCE(bookmarks_view_list_table.value, 0)) AS bookmarks_view_list,
+    --FxA
+    SUM(COALESCE(metrics.counter.sync_create_account_pressed, 0)) AS sync_create_account_pressed,
+    SUM(COALESCE(metrics.counter.sync_open_tab, 0)) AS sync_open_tab,
+    SUM(COALESCE(metrics.counter.sync_sign_in_sync_pressed, 0)) AS sync_sign_in_sync_pressed,
+    --Privacy
+    MAX(COALESCE(metrics.quantity.tabs_private_tabs_quantity, 0)) AS tabs_private_tabs_quantity,
+    CAST(
+      MAX(COALESCE(metrics.boolean.preferences_close_private_tabs, FALSE)) AS INT64
+    ) AS preferences_close_private_tabs,
+    CAST(
+      MAX(COALESCE(metrics.boolean.tracking_protection_enabled, FALSE)) AS INT64
+    ) AS tracking_protection_enabled,
+    CAST(
+      MAX(COALESCE(LOWER(metrics.string.tracking_protection_strength) = "strict", FALSE)) AS INT64
+    ) AS tracking_protection_strict_enabled,
+    --Tab Count
+    MAX(COALESCE(metrics.quantity.tabs_normal_tabs_quantity, 0)) AS tabs_normal_tabs_quantity,
+    MAX(COALESCE(metrics.quantity.tabs_inactive_tabs_count, 0)) AS tabs_inactive_tabs_count,
+    --Default Browser
+    SUM(
+      COALESCE(metrics.counter.app_opened_as_default_browser, 0)
+    ) AS app_opened_as_default_browser,
+    SUM(
+      COALESCE(metrics.counter.settings_menu_set_as_default_browser_pressed, 0)
+    ) AS settings_menu_set_as_default_browser_pressed,
+    --Notification
+    CAST(
+      MAX(COALESCE(metrics.boolean.preferences_sync_notifs, FALSE)) AS INT64
+    ) AS preferences_sync_notifs,
+    CAST(
+      MAX(COALESCE(metrics.boolean.preferences_tips_and_features_notifs, FALSE)) AS INT64
+    ) AS preferences_tips_and_features_notifs,
+    --Customize Home
+    CAST(
+      MAX(COALESCE(metrics.boolean.preferences_jump_back_in, FALSE)) AS INT64
+    ) AS preferences_jump_back_in,
+    CAST(
+      MAX(COALESCE(metrics.boolean.preferences_recently_visited, FALSE)) AS INT64
+    ) AS preferences_recently_visited,
+    CAST(
+      MAX(COALESCE(metrics.boolean.preferences_recently_saved, FALSE)) AS INT64
+    ) AS preferences_recently_saved,
+    CAST(MAX(COALESCE(metrics.boolean.preferences_pocket, FALSE)) AS INT64) AS preferences_pocket,
+    SUM(COALESCE(metrics.counter.app_menu_customize_homepage, 0)) AS app_menu_customize_homepage,
+    SUM(
+      COALESCE(metrics.counter.firefox_home_page_customize_homepage_button, 0)
+    ) AS firefox_home_page_customize_homepage_button,
+    --Address
+    MAX(COALESCE(metrics.quantity.addresses_saved_all, 0)) AS addresses_saved_all
+  FROM
+    metrics_dau
+  LEFT JOIN
+    UNNEST(metrics.labeled_counter.bookmarks_add) AS bookmarks_add_table
+  LEFT JOIN
+    UNNEST(metrics.labeled_counter.bookmarks_delete) AS bookmarks_delete_table
+  LEFT JOIN
+    UNNEST(metrics.labeled_counter.bookmarks_edit) AS bookmarks_edit_table
+  LEFT JOIN
+    UNNEST(metrics.labeled_counter.bookmarks_open) AS bookmarks_open_table
+  LEFT JOIN
+    UNNEST(metrics.labeled_counter.bookmarks_view_list) AS bookmarks_view_list_table
+  GROUP BY
+    dau_date,
+    client_id
+)
+-- Aggregated feature usage
+SELECT
+  @submission_date AS submission_date,
+  dau_date AS metric_date,
+  COUNT(DISTINCT client_id) AS clients,
+  channel,
+  country,
+  adjust_network,
+  is_default_browser,
+  /*Logins*/
+  -- logins_deleted
+  COUNT(DISTINCT IF(logins_deleted > 0, client_id, NULL)) AS logins_deleted_users,
+  SUM(logins_deleted) AS logins_deleted,
+  -- logins_modified
+  COUNT(DISTINCT IF(logins_modified > 0, client_id, NULL)) AS logins_modified_users,
+  SUM(logins_modified) AS logins_modified,
+  -- logins_saved
+  COUNT(DISTINCT IF(logins_saved > 0, client_id, NULL)) AS logins_saved_users,
+  SUM(logins_saved) AS logins_saved,
+  -- logins_saved_all
+  COUNT(DISTINCT IF(logins_saved_all > 0, client_id, NULL)) AS logins_saved_all_users,
+  SUM(logins_saved_all) AS logins_saved_all,
+  /*Credit Card*/
+  --credit card autofill enabled
+  COUNT(
+    DISTINCT IF(credit_card_autofill_enabled > 0, client_id, NULL)
+  ) AS credit_card_autofill_enabled_users,
+  SUM(credit_card_autofill_enabled) AS credit_card_autofill_enabled,
+  --credit_card_sync_enabled
+  COUNT(
+    DISTINCT IF(credit_card_sync_enabled > 0, client_id, NULL)
+  ) AS credit_card_sync_enabled_users,
+  SUM(credit_card_sync_enabled) AS credit_card_sync_enabled,
+  --credit_card_deleted
+  COUNT(DISTINCT IF(credit_card_deleted > 0, client_id, NULL)) AS credit_card_deleted_users,
+  SUM(credit_card_deleted) AS credit_card_deleted,
+  --credit_card_modified
+  COUNT(DISTINCT IF(credit_card_modified > 0, client_id, NULL)) AS credit_card_modified_users,
+  SUM(credit_card_modified) AS credit_card_modified,
+  --credit_card_saved
+  COUNT(DISTINCT IF(credit_card_saved > 0, client_id, NULL)) AS credit_card_saved_users,
+  SUM(credit_card_saved) AS credit_card_saved,
+  --credit_card_saved_all
+  COUNT(DISTINCT IF(credit_card_saved_all > 0, client_id, NULL)) AS credit_card_saved_all_users,
+  SUM(credit_card_saved_all) AS credit_card_saved_all,
+  /*Bookmark*/
+  --bookmarks_add
+  COUNT(DISTINCT IF(bookmarks_add > 0, client_id, NULL)) AS bookmarks_add_users,
+  SUM(bookmarks_add) AS bookmarks_add,
+  -- Bookmarks Delete
+  COUNT(DISTINCT IF(bookmarks_delete > 0, client_id, NULL)) AS bookmarks_delete_users,
+  SUM(bookmarks_delete) AS bookmarks_delete,
+  -- Bookmarks Edit
+  COUNT(DISTINCT IF(bookmarks_edit > 0, client_id, NULL)) AS bookmarks_edit_users,
+  SUM(bookmarks_edit) AS bookmarks_edit,
+  -- Has Mobile Bookmarks
+  COUNT(DISTINCT IF(has_mobile_bookmarks > 0, client_id, NULL)) AS has_mobile_bookmarks_users,
+  SUM(has_mobile_bookmarks) AS has_mobile_bookmarks,
+  -- Mobile Bookmarks Count
+  COUNT(DISTINCT IF(mobile_bookmarks_count > 0, client_id, NULL)) AS mobile_bookmarks_count_users,
+  SUM(mobile_bookmarks_count) AS mobile_bookmarks_count,
+  -- Bookmarks Open
+  COUNT(DISTINCT IF(bookmarks_open > 0, client_id, NULL)) AS bookmarks_open_users,
+  SUM(bookmarks_open) AS bookmarks_open,
+  -- Bookmarks View List
+  COUNT(DISTINCT IF(bookmarks_view_list > 0, client_id, NULL)) AS bookmarks_view_list_users,
+  SUM(bookmarks_view_list) AS bookmarks_view_list,
+  /*FxA*/
+  --sync_create_account_pressed
+  COUNT(
+    DISTINCT IF(sync_create_account_pressed > 0, client_id, NULL)
+  ) AS sync_create_account_pressed_users,
+  SUM(sync_create_account_pressed) AS sync_create_account_pressed,
+  --sync_open_tab
+  COUNT(DISTINCT IF(sync_open_tab > 0, client_id, NULL)) AS sync_open_tab_users,
+  SUM(sync_open_tab) AS sync_open_tab,
+  --sync_sign_in_sync_pressed
+  COUNT(
+    DISTINCT IF(sync_sign_in_sync_pressed > 0, client_id, NULL)
+  ) AS sync_sign_in_sync_pressed_users,
+  SUM(sync_sign_in_sync_pressed) AS sync_sign_in_sync_pressed,
+  /*Privacy*/
+  --tabs_private_tabs_quantity
+  COUNT(
+    DISTINCT IF(tabs_private_tabs_quantity > 0, client_id, NULL)
+  ) AS tabs_private_tabs_quantity_users,
+  SUM(tabs_private_tabs_quantity) AS tabs_private_tabs_quantity,
+  -- Preferences Close Private Tabs
+  COUNT(
+    DISTINCT IF(preferences_close_private_tabs > 0, client_id, NULL)
+  ) AS preferences_close_private_tabs_users,
+  SUM(preferences_close_private_tabs) AS preferences_close_private_tabs,
+  -- Tracking Protection Enabled
+  COUNT(
+    DISTINCT IF(tracking_protection_enabled > 0, client_id, NULL)
+  ) AS tracking_protection_enabled_users,
+  SUM(tracking_protection_enabled) AS tracking_protection_enabled,
+  -- Tracking Protection Strict
+  COUNT(
+    DISTINCT IF(tracking_protection_strict_enabled > 0, client_id, NULL)
+  ) AS tracking_protection_strict_users,
+  SUM(tracking_protection_strict_enabled) AS tracking_protection_strict,
+  /*Tab Count*/
+  --tabs_normal_tabs_quantity
+  COUNT(
+    DISTINCT IF(tabs_normal_tabs_quantity > 0, client_id, NULL)
+  ) AS tabs_normal_tabs_quantity_users,
+  SUM(tabs_normal_tabs_quantity) AS tabs_normal_tabs_quantity,
+  --tabs_inactive_tabs_count
+  COUNT(
+    DISTINCT IF(tabs_inactive_tabs_count > 0, client_id, NULL)
+  ) AS tabs_inactive_tabs_count_users,
+  SUM(tabs_inactive_tabs_count) AS tabs_inactive_tabs_count,
+  /*Default Browser*/
+  --app_opened_as_default_browser
+  COUNT(
+    DISTINCT IF(app_opened_as_default_browser > 0, client_id, NULL)
+  ) AS app_opened_as_default_browser_users,
+  SUM(app_opened_as_default_browser) AS app_opened_as_default_browser,
+  -- settings_menu_set_as_default_browser_pressed
+  COUNT(
+    DISTINCT IF(settings_menu_set_as_default_browser_pressed > 0, client_id, NULL)
+  ) AS settings_menu_set_as_default_browser_pressed_users,
+  SUM(settings_menu_set_as_default_browser_pressed) AS settings_menu_set_as_default_browser_pressed,
+  /*Notification*/
+  --preferences_sync_notifs
+  COUNT(DISTINCT IF(preferences_sync_notifs > 0, client_id, NULL)) AS preferences_sync_notifs_users,
+  SUM(preferences_sync_notifs) AS preferences_sync_notifs,
+  -- preferences_tips_and_features_notifs
+  COUNT(
+    DISTINCT IF(preferences_tips_and_features_notifs > 0, client_id, NULL)
+  ) AS preferences_tips_and_features_notifs_users,
+  SUM(preferences_tips_and_features_notifs) AS preferences_tips_and_features_notifs,
+  /*Customize Home*/
+  --preferences_jump_back_in
+  COUNT(
+    DISTINCT IF(preferences_jump_back_in > 0, client_id, NULL)
+  ) AS preferences_jump_back_in_users,
+  SUM(preferences_jump_back_in) AS preferences_jump_back_in,
+  -- Preferences Recently Visited
+  COUNT(
+    DISTINCT IF(preferences_recently_visited > 0, client_id, NULL)
+  ) AS preferences_recently_visited_users,
+  SUM(preferences_recently_visited) AS preferences_recently_visited,
+  -- Preferences Recently Saved
+  COUNT(
+    DISTINCT IF(preferences_recently_saved > 0, client_id, NULL)
+  ) AS preferences_recently_saved_users,
+  SUM(preferences_recently_saved) AS preferences_recently_saved,
+  -- Preferences Pocket
+  COUNT(DISTINCT IF(preferences_pocket > 0, client_id, NULL)) AS preferences_pocket_users,
+  SUM(preferences_pocket) AS preferences_pocket,
+  -- App Menu Customize Homepage
+  COUNT(
+    DISTINCT IF(app_menu_customize_homepage > 0, client_id, NULL)
+  ) AS app_menu_customize_homepage_users,
+  SUM(app_menu_customize_homepage) AS app_menu_customize_homepage,
+  -- Firefox Home Page Customize Homepage Button
+  COUNT(
+    DISTINCT IF(firefox_home_page_customize_homepage_button > 0, client_id, NULL)
+  ) AS firefox_home_page_customize_homepage_button_users,
+  SUM(firefox_home_page_customize_homepage_button) AS firefox_home_page_customize_homepage_button,
+  -- addresses_saved_all
+  COUNT(DISTINCT IF(addresses_saved_all > 0, client_id, NULL)) AS addresses_saved_all_users,
+  SUM(addresses_saved_all) AS addresses_saved_all
+FROM
+  metric_ping_clients_feature_usage
+LEFT JOIN
+  client_attribution
+  USING (client_id)
+GROUP BY
+  submission_date,
+  metric_date,
+  channel,
+  country,
+  adjust_network,
+  is_default_browser

sql/moz-fx-data-shared-prod/firefox_ios_derived/feature_usage_metrics_v2/schema.yaml

@@ -0,0 +1,325 @@
+fields:
+- name: submission_date
+  type: DATE
+  mode: NULLABLE
+  description: Airflow's internal execution_date.
+- name: metric_date
+  type: DATE
+  mode: NULLABLE
+  description: reporting date for feature usage, aligned with DAU.
+- name: clients
+  type: INTEGER
+  mode: NULLABLE
+  description: number of clients accounted for in the aggregation.
+- name: channel
+  type: STRING
+  mode: NULLABLE
+  description: Channel
+- name: country
+  type: STRING
+  mode: NULLABLE
+  description: Country
+- name: adjust_network
+  type: STRING
+  mode: NULLABLE
+  description: Adjust Network
+- name: is_default_browser
+  type: BOOLEAN
+  mode: NULLABLE
+  description: Is Default Browser
+- name: logins_deleted_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Logins Deleted Users
+- name: logins_deleted
+  type: INTEGER
+  mode: NULLABLE
+  description: Logins Deleted
+- name: logins_modified_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Logins Modified Users
+- name: logins_modified
+  type: INTEGER
+  mode: NULLABLE
+  description: Logins Modified
+- name: logins_saved_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Logins Saved Users
+- name: logins_saved
+  type: INTEGER
+  mode: NULLABLE
+  description: Logins Saved
+- name: logins_saved_all_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Logins Saved All Users
+- name: logins_saved_all
+  type: INTEGER
+  mode: NULLABLE
+  description: Logins Saved All
+- name: credit_card_autofill_enabled_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Credit Card Autofill Enabled Users
+- name: credit_card_autofill_enabled
+  type: INTEGER
+  mode: NULLABLE
+  description: Credit Card Autofill Enabled
+- name: credit_card_sync_enabled_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Credit Card Sync Enabled Users
+- name: credit_card_sync_enabled
+  type: INTEGER
+  mode: NULLABLE
+  description: Credit Card Sync Enabled
+- name: credit_card_deleted_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Credit Card Deleted Users
+- name: credit_card_deleted
+  type: INTEGER
+  mode: NULLABLE
+  description: Credit Card Deleted
+- name: credit_card_modified_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Credit Card Modified Users
+- name: credit_card_modified
+  type: INTEGER
+  mode: NULLABLE
+  description: Credit Card Modified
+- name: credit_card_saved_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Credit Card Saved Users
+- name: credit_card_saved
+  type: INTEGER
+  mode: NULLABLE
+  description: Credit Card Saved
+- name: credit_card_saved_all_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Credit Card Saved All Users
+- name: credit_card_saved_all
+  type: INTEGER
+  mode: NULLABLE
+  description: Credit Card Saved All
+- name: bookmarks_add_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Bookmarks Add Users
+- name: bookmarks_add
+  type: INTEGER
+  mode: NULLABLE
+  description: Bookmarks Add
+- name: bookmarks_delete_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Bookmarks Delete Users
+- name: bookmarks_delete
+  type: INTEGER
+  mode: NULLABLE
+  description: Bookmarks Delete
+- name: bookmarks_edit_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Bookmarks Edit Users
+- name: bookmarks_edit
+  type: INTEGER
+  mode: NULLABLE
+  description: Bookmarks Edit
+- name: has_mobile_bookmarks_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Has Mobile Bookmarks Users
+- name: has_mobile_bookmarks
+  type: INTEGER
+  mode: NULLABLE
+  description: Has Mobile Bookmarks
+- name: mobile_bookmarks_count_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Mobile Bookmarks Count Users
+- name: mobile_bookmarks_count
+  type: INTEGER
+  mode: NULLABLE
+  description: Mobile Bookmarks Count
+- name: bookmarks_open_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Bookmarks Open Users
+- name: bookmarks_open
+  type: INTEGER
+  mode: NULLABLE
+  description: Bookmarks Open
+- name: bookmarks_view_list_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Bookmarks View List Users
+- name: bookmarks_view_list
+  type: INTEGER
+  mode: NULLABLE
+  description: Bookmarks View List
+- name: sync_create_account_pressed_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Sync Create Account Pressed Users
+- name: sync_create_account_pressed
+  type: INTEGER
+  mode: NULLABLE
+  description: Sync Create Account Pressed
+- name: sync_open_tab_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Sync Open Tab Users
+- name: sync_open_tab
+  type: INTEGER
+  mode: NULLABLE
+  description: Sync Open Tab
+- name: sync_sign_in_sync_pressed_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Sync Sign In Sync Pressed Users
+- name: sync_sign_in_sync_pressed
+  type: INTEGER
+  mode: NULLABLE
+  description: Sync Sign In Sync Pressed
+- name: tabs_private_tabs_quantity_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Tabs Private Tabs Quantity Users
+- name: tabs_private_tabs_quantity
+  type: INTEGER
+  mode: NULLABLE
+  description: Tabs Private Tabs Quantity
+- name: preferences_close_private_tabs_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Preferences Close Private Tabs Users
+- name: preferences_close_private_tabs
+  type: INTEGER
+  mode: NULLABLE
+  description: Preferences Close Private Tabs
+- name: tracking_protection_enabled_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Tracking Protection Enabled Users
+- name: tracking_protection_enabled
+  type: INTEGER
+  mode: NULLABLE
+  description: Tracking Protection Enabled
+- name: tracking_protection_strict_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Tracking Protection Strict Users
+- name: tracking_protection_strict
+  type: INTEGER
+  mode: NULLABLE
+  description: Tracking Protection Strict
+- name: tabs_normal_tabs_quantity_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Tabs Normal Tabs Quantity Users
+- name: tabs_normal_tabs_quantity
+  type: INTEGER
+  mode: NULLABLE
+  description: Tabs Normal Tabs Quantity
+- name: tabs_inactive_tabs_count_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Tabs Inactive Tabs Count Users
+- name: tabs_inactive_tabs_count
+  type: INTEGER
+  mode: NULLABLE
+  description: Tabs Inactive Tabs Count
+- name: app_opened_as_default_browser_users
+  type: INTEGER
+  mode: NULLABLE
+  description: App Opened as Default Browser Users
+- name: app_opened_as_default_browser
+  type: INTEGER
+  mode: NULLABLE
+  description: App Opened as Default Browser
+- name: settings_menu_set_as_default_browser_pressed_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Settings Menu Set as Default Browser Pressed Users
+- name: settings_menu_set_as_default_browser_pressed
+  type: INTEGER
+  mode: NULLABLE
+  description: Settings Menu Set as Default Browser Pressed
+- name: preferences_sync_notifs_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Preferences Sync Notifs Users
+- name: preferences_sync_notifs
+  type: INTEGER
+  mode: NULLABLE
+  description: preferences_sync_notifs
+- name: preferences_tips_and_features_notifs_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Preferences Tips & Features Notifs Users
+- name: preferences_tips_and_features_notifs
+  type: INTEGER
+  mode: NULLABLE
+  description: Preferences Tips & Features Notifs
+- name: preferences_jump_back_in_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Preferences Jump Back in Users
+- name: preferences_jump_back_in
+  type: INTEGER
+  mode: NULLABLE
+  description: Preferences Jump Back In
+- name: preferences_recently_visited_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Preferences Recently Visited Users
+- name: preferences_recently_visited
+  type: INTEGER
+  mode: NULLABLE
+  description: Preferences Recently Visited
+- name: preferences_recently_saved_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Preferences Recently Saved Users
+- name: preferences_recently_saved
+  type: INTEGER
+  mode: NULLABLE
+  description: Preferences Recently Saved
+- name: preferences_pocket_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Preferences Pocket Users
+- name: preferences_pocket
+  type: INTEGER
+  mode: NULLABLE
+  description: Preferences Pocket
+- name: app_menu_customize_homepage_users
+  type: INTEGER
+  mode: NULLABLE
+  description: App Menu Customize Homepage Users
+- name: app_menu_customize_homepage
+  type: INTEGER
+  mode: NULLABLE
+  description: App Menu Customize Homepage
+- name: firefox_home_page_customize_homepage_button_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Firefox Home Page Customize Homepage Button Users
+- name: firefox_home_page_customize_homepage_button
+  type: INTEGER
+  mode: NULLABLE
+  description: Firefox Home Page Customize Homepage Button
+- name: addresses_saved_all_users
+  type: INTEGER
+  mode: NULLABLE
+  description: Addresses Saved All Users
+- name: addresses_saved_all
+  type: INTEGER
+  mode: NULLABLE
+  description: Addresses Saved All

sql/moz-fx-data-shared-prod/firefox_ios_derived/new_profile_activation_v1/query.sql

@@ -35,10 +35,9 @@ client_search AS (
     USING (client_id)
   WHERE
     (submission_date BETWEEN DATE_SUB(@submission_date, INTERVAL 3 DAY) AND @submission_date)
-    AND normalized_app_name = 'Fennec'
-    AND os = 'iOS'
+    AND normalized_app_name_os = "Firefox iOS"
   GROUP BY
-    1
+    client_id
 ),
 dou AS (
   SELECT
@@ -70,7 +69,7 @@ SELECT
   first_seen_date,
   submission_date,
   1 AS new_profile,
-  CAST(days_2_7 > 1 AND COALESCE(search_count, 0) > 0 AS integer) AS activated
+  CAST(days_2_7 > 1 AND COALESCE(search_count, 0) > 0 AS INTEGER) AS activated
 FROM
   dou
 INNER JOIN

sql/moz-fx-data-shared-prod/firefox_ios_derived/new_profile_activation_v2/query.sql

@@ -27,8 +27,7 @@ client_search AS (
     `moz-fx-data-shared-prod.search.mobile_search_clients_daily`
   WHERE
     (submission_date BETWEEN DATE_SUB(@submission_date, INTERVAL 3 DAY) AND @submission_date)
-    AND os = 'iOS'
-    AND normalized_app_name = 'Fennec'
+    AND normalized_app_name_os = 'Firefox iOS'
   GROUP BY
     client_id
 ),

sql/moz-fx-data-shared-prod/google_ads_derived/android_app_campaign_stats_v1/bigconfig.yml

@@ -1,61 +1,60 @@
 type: BIGCONFIG_FILE
-
 tag_deployments:
-  - collection:
-      name: Operational Checks
-      notification_channels:
-        - slack: '#de-bigeye-triage'
-    deployments:
-      - column_selectors:
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.date
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.campaign_region
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.campaign_country_code
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.campaign_language
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.ad_group
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.impressions
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.clicks
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.new_profiles
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.activated_profiles
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.repeat_users
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.week_4_retained_users
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.spend
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.lifetime_value
-        metrics:
-          - saved_metric_id: is_not_null
-            lookback:
-              lookback_type: DATA_TIME
-              lookback_window:
-                interval_type: DAYS
-                interval_value: 28
-            rct_overrides:
-              - date
-      - column_selectors:
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.campaign_country_code
-        metrics:
-          - saved_metric_id: is_2_char_len
-            lookback:
-              lookback_type: DATA_TIME
-              lookback_window:
-                interval_type: DAYS
-                interval_value: 28
-            rct_overrides:
-              - date
-      - column_selectors:
-          - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.*
-        metrics:
-          - saved_metric_id: volume
-            lookback:
-              lookback_type: DATA_TIME
-              lookback_window:
-                interval_type: DAYS
-                interval_value: 28
-            rct_overrides:
-              - date
-          - saved_metric_id: freshness
-            lookback:
-              lookback_type: DATA_TIME
-              lookback_window:
-                interval_type: DAYS
-                interval_value: 28
-            rct_overrides:
-              - date
+- collection:
+    name: Operational Checks
+    notification_channels:
+    - slack: '#de-bigeye-triage'
+  deployments:
+  - column_selectors:
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.date
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.campaign_region
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.campaign_country_code
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.campaign_language
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.ad_group
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.impressions
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.clicks
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.new_profiles
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.activated_profiles
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.repeat_users
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.week_4_retained_users
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.spend
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.lifetime_value
+    metrics:
+    - saved_metric_id: is_not_null
+      lookback:
+        lookback_window:
+          interval_type: DAYS
+          interval_value: 28
+        lookback_type: DATA_TIME
+      rct_overrides:
+      - date
+  - column_selectors:
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.campaign_country_code
+    metrics:
+    - saved_metric_id: is_2_char_len
+      lookback:
+        lookback_window:
+          interval_type: DAYS
+          interval_value: 28
+        lookback_type: DATA_TIME
+      rct_overrides:
+      - date
+  - column_selectors:
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.*
+    metrics:
+    - saved_metric_id: volume
+      lookback:
+        lookback_window:
+          interval_type: DAYS
+          interval_value: 28
+        lookback_type: DATA_TIME
+      rct_overrides:
+      - date
+    - saved_metric_id: freshness
+      lookback:
+        lookback_window:
+          interval_type: DAYS
+          interval_value: 28
+        lookback_type: DATA_TIME
+      rct_overrides:
+      - date

sql/moz-fx-data-shared-prod/google_ads_derived/android_app_campaign_stats_v1/metadata.yaml

@@ -12,6 +12,8 @@ owners:
 labels:
   incremental: true
   owner1: kwindau@mozilla.com
+  table_type: aggregate
+  shredder_mitigation: true
 scheduling:
   dag_name: bqetl_fivetran_google_ads
   depends_on_past: false

sql/moz-fx-data-shared-prod/jira_service_desk/field_option/metadata.yaml

@@ -0,0 +1,4 @@
+---
+friendly_name: Jira Service Desk Field Options
+description: |-
+  Jira field options for Service Desk synced via Fivetran.

sql/moz-fx-data-shared-prod/jira_service_desk/field_option/view.sql

@@ -0,0 +1,7 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.jira_service_desk.field_option`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod.jira_service_desk_syndicate.field_option`

sql/moz-fx-data-shared-prod/jira_service_desk/issue/view.sql

@@ -1,7 +1,69 @@
 CREATE OR REPLACE VIEW
   `moz-fx-data-shared-prod.jira_service_desk.issue`
 AS
+WITH requests AS (
+  -- Join requests related data
+  SELECT
+    * EXCEPT (id, description),
+    id AS request_id,
+    description AS request_description
+  FROM
+    (
+      SELECT
+        * EXCEPT (_fivetran_synced)
+      FROM
+        `moz-fx-data-shared-prod.jira_service_desk_syndicate.request`
+    ) AS request
+  LEFT JOIN
+    (
+      SELECT
+        * EXCEPT (_fivetran_synced, service_desk_id)
+      FROM
+        `moz-fx-data-shared-prod.jira_service_desk_syndicate.request_type`
+    ) AS request_type
+    ON request.request_type_id = request_type.id
+),
+fields AS (
+  -- Join fields related data
+  SELECT
+    * EXCEPT (id, field_id, description),
+    id AS field_id,
+    description AS field_description
+  FROM
+    (
+      SELECT
+        * EXCEPT (_fivetran_synced, _fivetran_deleted, name),
+        name AS field_name
+      FROM
+        `moz-fx-data-shared-prod.jira_service_desk_syndicate.field`
+    ) AS field
+  LEFT JOIN
+    (
+      SELECT
+        * EXCEPT (_fivetran_synced)
+      FROM
+        `moz-fx-data-shared-prod.jira_service_desk_syndicate.issue_field_history`
+    ) AS field_history
+    ON field_history.field_id = field.id
+  LEFT JOIN
+    (
+      SELECT
+        * EXCEPT (_fivetran_synced, id, name),
+        id AS field_option_id,
+        name AS field_option_name
+      FROM
+        `moz-fx-data-shared-prod.jira_service_desk_syndicate.field_option`
+    ) AS field_option
+    ON field_history.value = SAFE_CAST(field_option.field_option_id AS STRING)
+)
+-- Join requests and fields data with issues
 SELECT
-  *
+  * EXCEPT (issue_id, parent_id)
 FROM
-  `moz-fx-data-shared-prod.jira_service_desk_syndicate.issue`
+  `moz-fx-data-shared-prod.jira_service_desk_syndicate.issue` AS issue
+LEFT JOIN
+  requests
+  ON issue.id = requests.issue_id
+LEFT JOIN
+  fields
+  ON issue.id = fields.issue_id

sql/moz-fx-data-shared-prod/jira_service_desk_derived/user_v1/query.py

@@ -32,6 +32,10 @@ class BigQueryAPI:
                 bigquery.SchemaField("account_id", "STRING"),
                 bigquery.SchemaField("account_status", "STRING"),                
                 bigquery.SchemaField("name", "STRING"),
+                bigquery.SchemaField("time_zone", "STRING"),
+                bigquery.SchemaField("email_address", "STRING"),
+                bigquery.SchemaField("account_type", "STRING"),
+                bigquery.SchemaField("locale", "STRING"),
             ],
             autodetect=False,
             write_disposition=write_disposition,
@@ -44,6 +48,7 @@ class BigQueryAPI:
         job.result()
 
 
+
 class JiraAPI:
     def __init__(self, args) -> None:
         self.logger = logging.getLogger(self.__class__.__name__)
@@ -56,7 +61,7 @@ class JiraAPI:
         self.base_jira_url = args.base_jira_url
         self.auth = HTTPBasicAuth(self.secrets_dict.get('jira_username'), self.secrets_dict.get('jira_token'))
 
-    def get_users_paged(self, max_results=300):
+    def get_users_paged(self, max_results=500):
         startAt = 0
         headers = {"Accept": "application/json"}
 
@@ -87,6 +92,10 @@ class JiraAPI:
                         "active" if user.get("active", "") else "inactive"
                     ),                    
                     "name": user.get("displayName", ""),
+                    "time_zone": user.get("timeZone", ""),                    
+                    "email_address": user.get("emailAddress", ""),
+                    "account_type": user.get("accountType", ""),
+                    "locale": user.get("locale", ""),                    
                 }
                 for user in users
             ] 

sql/moz-fx-data-shared-prod/jira_service_desk_derived/user_v1/schema.yaml

@@ -8,3 +8,15 @@ fields:
 - name: name
   type: STRING
   mode: NULLABLE
+- name: time_zone
+  type: STRING
+  mode: NULLABLE
+- name: email_address
+  type: STRING
+  mode: NULLABLE
+- name: account_type
+  type: STRING
+  mode: NULLABLE
+- name: locale
+  type: STRING
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/jira_service_desk_syndicate/dataset_metadata.yaml

@@ -15,6 +15,7 @@ syndication:
     - status
     - resolution
     - field
+    - field_option
     - issue_field_history
     - issue_multiselect_history
     - status_category

sql/moz-fx-data-shared-prod/legal_derived/dataset_metadata.yaml

@@ -0,0 +1,10 @@
+friendly_name: Legal tables
+description: |-
+  Reference data
+dataset_base_acl: derived
+user_facing: false
+labels: {}
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential

sql/moz-fx-data-shared-prod/mlhackweek_search/dataset_metadata.yaml

@@ -1,11 +0,0 @@
-friendly_name: mlhackweek-search
-# yamllint disable rule:line-length
-description: |-
-  User-facing views related to document namespace mlhackweek-search; see https://github.com/mozilla-services/mozilla-pipeline-schemas/tree/generated-schemas/schemas/mlhackweek-search
-dataset_base_acl: view_restricted
-user_facing: true
-labels: {}
-workgroup_access:
-- role: roles/bigquery.dataViewer
-  members:
-  - workgroup:mlhackweek-search

sql/moz-fx-data-shared-prod/mlhackweek_search_derived/dataset_metadata.yaml

@@ -1,11 +0,0 @@
-friendly_name: mlhackweek-search Derived
-# yamllint disable rule:line-length
-description: |-
-  Derived tables related to document namespace mlhackweek-search, usually populated via queries defined in https://github.com/mozilla/bigquery-etl and managed by Airflow
-dataset_base_acl: derived_restricted
-user_facing: false
-labels: {}
-workgroup_access:
-- role: roles/bigquery.dataViewer
-  members:
-  - workgroup:mlhackweek-search

sql/moz-fx-data-shared-prod/monitoring/bigeye_usage/view.sql

@@ -8,6 +8,11 @@ SELECT
   reference_table_id,
   creation_date,
   task_duration,
+  EXTRACT(HOUR FROM task_duration) * 3600 + EXTRACT(MINUTE FROM task_duration) * 60 + EXTRACT(
+    SECOND
+    FROM
+      task_duration
+  ) AS task_duration_seconds,
   total_terabytes_processed,
   total_terabytes_billed,
   total_slot_ms,

sql/moz-fx-data-shared-prod/monitoring/event_counts_glean/metadata.yaml

@@ -0,0 +1,5 @@
+friendly_name: Event Counts Glean
+description: |-
+  Daily event counts for all Glean apps, derived from monitoring_derived.event_monitoring_aggregates_v1
+owners:
+- bewu@mozilla.com