GLAM purge percentile calculations and prep downstream (#5966)

* Remove percentiles

* Remove tests that test percentiles

* Refresh scripts insert null to new percentiles

* Remove percentile columns from queries and schemas

* Delete more percentile tables

* Formatting

* histogram_cast_struct's keys are strings

* Re-add test after fixing failure cause

bigquery_etl/glam/generate.py

@@ -291,13 +291,6 @@ def main():
             ),
             channel=channel_prefixes[args.prefix],
         ),
-        table(
-            "scalar_percentiles_v1",
-            **models.scalar_percentiles(
-                source_table=f"glam_etl.{args.prefix}__clients_scalar_aggregates_v1"
-            ),
-        ),
-        table("histogram_percentiles_v1"),
         view("view_probe_counts_v1"),
         view("view_user_counts_v1", **models.user_counts()),
         view(

bigquery_etl/glam/models.py

@@ -154,26 +154,6 @@ def probe_counts(**kwargs):
     )
 
 
-def scalar_percentiles(**kwargs):
-    """Variables for scalar percentiles."""
-    attributes = ["ping_type", "os", "app_version", "app_build_id", "channel"]
-    fixed_attributes = ["app_version", "channel"]
-    cubed_attributes = [x for x in attributes if x not in fixed_attributes]
-
-    return dict(
-        # TODO: be consistent with naming of attributes (e.g. attributes_list)
-        attributes=attributes,
-        cubed_attributes=cubed_attributes,
-        attribute_combinations=compute_datacube_groupings(cubed_attributes),
-        aggregate_attributes="""
-            metric,
-            metric_type,
-            key
-        """,
-        **kwargs,
-    )
-
-
 def user_counts(**kwargs):
     """Variables for user counts."""
     attributes = ["ping_type", "os", "app_version", "app_build_id", "channel"]

bigquery_etl/glam/templates/extract_probe_counts_v1.sql

@@ -20,15 +20,9 @@ WITH final_probe_extract AS (
     client_agg_type,
     MAX(total_users) AS total_users,
     MAX(IF(agg_type = "histogram", mozfun.glam.histogram_cast_json(aggregates), NULL)) AS histogram,
-    MAX(
-      IF(agg_type = "percentiles", mozfun.glam.histogram_cast_json(aggregates), NULL)
-    ) AS percentiles,
     MAX(
       IF(agg_type = "histogram", mozfun.glam.histogram_cast_json(non_norm_aggregates), NULL)
-    ) AS non_norm_histogram,
-    MAX(
-      IF(agg_type = "percentiles", mozfun.glam.histogram_cast_json(non_norm_aggregates), NULL)
-    ) AS non_norm_percentiles
+    ) AS non_norm_histogram
   FROM
     `{{ dataset }}.{{ prefix }}__view_probe_counts_v1`
   WHERE
@@ -87,9 +81,7 @@ ranked_data AS (
     cp.metric_type,
     total_users,
     histogram,
-    percentiles,
     non_norm_histogram,
-    non_norm_percentiles,
     CASE
       WHEN client_agg_type = ''
         THEN 0
@@ -106,9 +98,7 @@ ranked_data AS (
         cp.client_agg_type,
         cp.metric_type,
         histogram,
-        percentiles,
         non_norm_histogram,
-        non_norm_percentiles
       ORDER BY
         total_users,
         total_sample DESC
@@ -139,9 +129,7 @@ SELECT
   client_agg_type,
   total_users,
   histogram,
-  percentiles,
   non_norm_histogram,
-  non_norm_percentiles,
   CAST(total_sample AS INT) total_sample
 FROM
   ranked_data

bigquery_etl/glam/templates/histogram_percentiles_v1.sql

@@ -1,27 +0,0 @@
-{{ header }}
-SELECT
-  * EXCEPT (aggregates, non_norm_aggregates) REPLACE('percentiles' AS agg_type),
-  ARRAY<STRUCT<key STRING, value FLOAT64>>[
-    ('0.1', mozfun.glam.percentile(0.1, aggregates, metric_type)),
-    ('1', mozfun.glam.percentile(1, aggregates, metric_type)),
-    ('5', mozfun.glam.percentile(5, aggregates, metric_type)),
-    ('25', mozfun.glam.percentile(25, aggregates, metric_type)),
-    ('50', mozfun.glam.percentile(50, aggregates, metric_type)),
-    ('75', mozfun.glam.percentile(75, aggregates, metric_type)),
-    ('95', mozfun.glam.percentile(95, aggregates, metric_type)),
-    ('99', mozfun.glam.percentile(99, aggregates, metric_type)),
-    ('99.9', mozfun.glam.percentile(99.9, aggregates, metric_type))
-  ] AS aggregates,
-  ARRAY<STRUCT<key STRING, value FLOAT64>>[
-    ('0.1', mozfun.glam.percentile(0.1, non_norm_aggregates, metric_type)),
-    ('1', mozfun.glam.percentile(1, non_norm_aggregates, metric_type)),
-    ('5', mozfun.glam.percentile(5, non_norm_aggregates, metric_type)),
-    ('25', mozfun.glam.percentile(25, non_norm_aggregates, metric_type)),
-    ('50', mozfun.glam.percentile(50, non_norm_aggregates, metric_type)),
-    ('75', mozfun.glam.percentile(75, non_norm_aggregates, metric_type)),
-    ('95', mozfun.glam.percentile(95, non_norm_aggregates, metric_type)),
-    ('99', mozfun.glam.percentile(99, non_norm_aggregates, metric_type)),
-    ('99.9', mozfun.glam.percentile(99.9, non_norm_aggregates, metric_type))
-  ] AS non_norm_aggregates
-FROM
-  glam_etl.{{ prefix }}__histogram_probe_counts_v1

bigquery_etl/glam/templates/scalar_percentiles_v1.sql

@@ -1,46 +0,0 @@
-{{ header }}
-{% from 'macros.sql' import enumerate_table_combinations %}
-
-WITH flat_clients_scalar_aggregates AS (
-  SELECT
-    * EXCEPT (scalar_aggregates)
-  FROM
-    {{ source_table }}
-  CROSS JOIN
-    UNNEST(scalar_aggregates)
-),
-{{
-    enumerate_table_combinations(
-        "flat_clients_scalar_aggregates",
-        "all_combos",
-        cubed_attributes,
-        attribute_combinations
-    )
-}},
-pre_percentiles AS (
-    SELECT
-        {{ attributes | join(",") }},
-        {{ aggregate_attributes }},
-        agg_type AS client_agg_type,
-        'percentiles' AS agg_type,
-        APPROX_COUNT_DISTINCT(client_id) AS total_users,
-        APPROX_QUANTILES(value, 1000) AS aggregates
-    FROM
-        all_combos
-    GROUP BY
-        {{ attributes | join(",") }},
-        {{ aggregate_attributes }},
-        client_agg_type
-),
-percentiles AS (
-  SELECT
-    * REPLACE (mozfun.glam.map_from_array_offsets_precise([0.1, 1.0, 5.0, 25.0, 50.0, 75.0, 95.0, 99.0, 99.9], aggregates) AS aggregates)
-  FROM
-    pre_percentiles
-)
-SELECT
-  *,
-  --Scalars are always non normalized.
-  --This is to comply with histograms' schema
-  aggregates AS non_norm_aggregates
-FROM percentiles

bigquery_etl/glam/templates/view_probe_counts_v1.sql

@@ -12,16 +12,6 @@ WITH all_counts AS (
     *
   FROM
     `{{ project }}.{{ dataset }}.{{ prefix }}__histogram_probe_counts_v1`
-  UNION ALL
-  SELECT
-    *
-  FROM
-    `{{ project }}.{{ dataset }}.{{ prefix }}__scalar_percentiles_v1`
-  UNION ALL
-  SELECT
-    *
-  FROM
-    `{{ project }}.{{ dataset }}.{{ prefix }}__histogram_percentiles_v1`
 )
 SELECT
   *

bqetl_project.yaml

@@ -174,8 +174,6 @@ dry_run:
   - sql/moz-fx-data-shared-prod/telemetry_derived/client_probe_counts/view.sql
   - sql/moz-fx-data-shared-prod/telemetry_derived/glam_sample_counts_v1/query.sql
   - sql/moz-fx-data-shared-prod/telemetry_derived/glam_user_counts_v1/query.sql
-  - sql/moz-fx-data-shared-prod/telemetry_derived/scalar_percentiles_v1/query.sql
-  - sql/moz-fx-data-shared-prod/telemetry_derived/histogram_percentiles_v1/query.sql
   - sql/moz-fx-data-shared-prod/telemetry_derived/clients_scalar_probe_counts_v1/query.sql
   - sql/moz-fx-data-shared-prod/telemetry/client_probe_counts/view.sql
   - sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_etl_sql_run_check_v1/query.sql

script/glam/run_glam_sql

@@ -170,7 +170,6 @@ function run_desktop_sql {
         run_init "clients_scalar_aggregates_v1"
         run_query "clients_scalar_aggregates_v1"
         run_query "clients_scalar_probe_counts_v1"
-        run_query "scalar_percentiles_v1"
     fi
 
     # Run the rest of the clients histogram pipeline
@@ -184,7 +183,6 @@ function run_desktop_sql {
     run_partitioned_query "clients_histogram_bucket_counts_v1" 10 false
     run_query "clients_histogram_probe_counts_v1"
 
-    run_query "histogram_percentiles_v1"
     run_query "glam_user_counts_v1"
     run_query "glam_sample_counts_v1"
 }
@@ -281,12 +279,6 @@ function run_glean_sql {
             pids+=($!)
             wait_pids "${pids[@]}"
 
-            pids=()
-            run_query "${product}__scalar_percentiles_v1" &
-            pids+=($!)
-            run_query "${product}__histogram_percentiles_v1" &
-            pids+=($!)
-            wait_pids "${pids[@]}"
         fi
         pids=()
         run_view "${product}__view_probe_counts_v1" &

sql/glam-fenix-dev/glam_etl/firefox_desktop_glam_nightly__extract_probe_counts_v1/query.sql

@@ -19,9 +19,6 @@ SELECT
   client_agg_type,
   MAX(total_users) AS total_users,
   MAX(IF(agg_type = "histogram", mozfun.glam.histogram_cast_json(aggregates), NULL)) AS histogram,
-  MAX(
-    IF(agg_type = "percentiles", mozfun.glam.histogram_cast_json(aggregates), NULL)
-  ) AS percentiles,
 FROM
   `glam-fenix-dev.glam_etl.firefox_desktop_glam_nightly__view_probe_counts_v1`
 WHERE

sql/glam-fenix-dev/glam_etl/firefox_desktop_glam_nightly__histogram_percentiles_v1/metadata.yaml

@@ -1,6 +0,0 @@
-friendly_name: |-
-  Glam Etl - Firefox Desktop Glam Nightly - Histogram Percentiles
-description: |-
-  [DESCRIPTION_MISSING]
-owners:
-- efilho@mozilla.com

sql/glam-fenix-dev/glam_etl/firefox_desktop_glam_nightly__histogram_percentiles_v1/query.sql

@@ -1,14 +0,0 @@
--- query for firefox_desktop_glam_nightly__histogram_percentiles_v1;
-SELECT
-  * EXCEPT (aggregates) REPLACE('percentiles' AS agg_type),
-  ARRAY<STRUCT<key STRING, value FLOAT64>>[
-    ('5', mozfun.glam.percentile(5, aggregates, metric_type)),
-    ('25', mozfun.glam.percentile(25, aggregates, metric_type)),
-    ('50', mozfun.glam.percentile(50, aggregates, metric_type)),
-    ('75', mozfun.glam.percentile(75, aggregates, metric_type)),
-    ('95', mozfun.glam.percentile(95, aggregates, metric_type)),
-    ('99', mozfun.glam.percentile(99, aggregates, metric_type)),
-    ('99.9', mozfun.glam.percentile(99.9, aggregates, metric_type))
-  ] AS aggregates
-FROM
-  `glam-fenix-dev.glam_etl.firefox_desktop_glam_nightly__histogram_probe_counts_v1`

sql/glam-fenix-dev/glam_etl/firefox_desktop_glam_nightly__scalar_percentiles_v1/metadata.yaml

@@ -1,6 +0,0 @@
-friendly_name: |-
-  Glam Etl - Firefox Desktop Glam Nightly - Scalar Percentiles
-description: |-
-  [DESCRIPTION_MISSING]
-owners:
-- efilho@mozilla.com

sql/glam-fenix-dev/glam_etl/firefox_desktop_glam_nightly__scalar_percentiles_v1/query.sql

@@ -1,78 +0,0 @@
--- query for firefox_desktop_glam_nightly__scalar_percentiles_v1;
-WITH flat_clients_scalar_aggregates AS (
-  SELECT
-    * EXCEPT (scalar_aggregates)
-  FROM
-    `glam-fenix-dev.glam_etl.firefox_desktop_glam_nightly__clients_scalar_aggregates_v1`
-  CROSS JOIN
-    UNNEST(scalar_aggregates)
-),
--- Cross join with the attribute combinations to reduce the query complexity
--- with respect to the number of operations. A table with n rows cross joined
--- with a combination of m attributes will generate a new table with n*m rows.
--- The glob ("*") symbol can be understood as selecting all of values belonging
--- to that group.
-static_combos AS (
-  SELECT
-    combos.*
-  FROM
-    UNNEST(
-      ARRAY<STRUCT<ping_type STRING, os STRING, app_build_id STRING>>[
-        (NULL, NULL, NULL),
-        (NULL, NULL, "*"),
-        (NULL, "*", NULL),
-        ("*", NULL, NULL),
-        (NULL, "*", "*"),
-        ("*", NULL, "*"),
-        ("*", "*", NULL),
-        ("*", "*", "*")
-      ]
-    ) AS combos
-),
-all_combos AS (
-  SELECT
-    table.* EXCEPT (ping_type, os, app_build_id),
-    COALESCE(combo.ping_type, table.ping_type) AS ping_type,
-    COALESCE(combo.os, table.os) AS os,
-    COALESCE(combo.app_build_id, table.app_build_id) AS app_build_id
-  FROM
-    flat_clients_scalar_aggregates table
-  CROSS JOIN
-    static_combos combo
-),
-percentiles AS (
-  SELECT
-    ping_type,
-    os,
-    app_version,
-    app_build_id,
-    channel,
-    metric,
-    metric_type,
-    key,
-    agg_type AS client_agg_type,
-    'percentiles' AS agg_type,
-    COUNT(DISTINCT(client_id)) AS total_users,
-    APPROX_QUANTILES(value, 1000) AS aggregates
-  FROM
-    all_combos
-  GROUP BY
-    ping_type,
-    os,
-    app_version,
-    app_build_id,
-    channel,
-    metric,
-    metric_type,
-    key,
-    client_agg_type
-)
-SELECT
-  * REPLACE (
-    mozfun.glam.map_from_array_offsets_precise(
-      [5.0, 25.0, 50.0, 75.0, 95.0, 99.0, 99.9],
-      aggregates
-    ) AS aggregates
-  )
-FROM
-  percentiles

sql/glam-fenix-dev/glam_etl/firefox_desktop_glam_nightly__view_probe_counts_v1/view.sql

@@ -12,16 +12,6 @@ WITH all_counts AS (
     *
   FROM
     `glam-fenix-dev.glam_etl.firefox_desktop_glam_nightly__histogram_probe_counts_v1`
-  UNION ALL
-  SELECT
-    *
-  FROM
-    `glam-fenix-dev.glam_etl.firefox_desktop_glam_nightly__scalar_percentiles_v1`
-  UNION ALL
-  SELECT
-    *
-  FROM
-    `glam-fenix-dev.glam_etl.firefox_desktop_glam_nightly__histogram_percentiles_v1`
 )
 SELECT
   *

sql/glam-fenix-dev/glam_etl/org_mozilla_fenix_glam_nightly__extract_probe_counts_v1/query.sql

@@ -18,10 +18,7 @@ SELECT
   SUBSTR(REPLACE(key, r"\x00", ""), 0, 200) AS metric_key,
   client_agg_type,
   MAX(total_users) AS total_users,
-  MAX(IF(agg_type = "histogram", mozfun.glam.histogram_cast_json(aggregates), NULL)) AS histogram,
-  MAX(
-    IF(agg_type = "percentiles", mozfun.glam.histogram_cast_json(aggregates), NULL)
-  ) AS percentiles,
+  MAX(IF(agg_type = "histogram", mozfun.glam.histogram_cast_json(aggregates), NULL)) AS histogram
 FROM
   `glam-fenix-dev.glam_etl.org_mozilla_fenix_glam_nightly__view_probe_counts_v1`
 WHERE

sql/glam-fenix-dev/glam_etl/org_mozilla_fenix_glam_nightly__extract_probe_counts_v1/schema.yaml

@@ -35,6 +35,3 @@ fields:
 - mode: NULLABLE
   name: histogram
   type: STRING
-- mode: NULLABLE
-  name: percentiles
-  type: STRING

sql/glam-fenix-dev/glam_etl/org_mozilla_fenix_glam_nightly__histogram_percentiles_v1/metadata.yaml

@@ -1,6 +0,0 @@
-friendly_name: |-
-  Glam Etl - Org Mozilla Fenix Glam Nightly - Histogram Percentiles
-description: |-
-  [DESCRIPTION_MISSING]
-owners:
-- efilho@mozilla.com

sql/glam-fenix-dev/glam_etl/org_mozilla_fenix_glam_nightly__histogram_percentiles_v1/query.sql

@@ -1,12 +0,0 @@
--- query for org_mozilla_fenix_glam_nightly__histogram_percentiles_v1;
-SELECT
-  * EXCEPT (aggregates) REPLACE('percentiles' AS agg_type),
-  ARRAY<STRUCT<key STRING, value FLOAT64>>[
-    ('5', mozfun.glam.percentile(5, aggregates, metric_type)),
-    ('25', mozfun.glam.percentile(25, aggregates, metric_type)),
-    ('50', mozfun.glam.percentile(50, aggregates, metric_type)),
-    ('75', mozfun.glam.percentile(75, aggregates, metric_type)),
-    ('95', mozfun.glam.percentile(95, aggregates, metric_type))
-  ] AS aggregates
-FROM
-  `glam-fenix-dev.glam_etl.org_mozilla_fenix_glam_nightly__histogram_probe_counts_v1`

sql/glam-fenix-dev/glam_etl/org_mozilla_fenix_glam_nightly__histogram_percentiles_v1/schema.yaml

@@ -1,44 +0,0 @@
-fields:
-- mode: NULLABLE
-  name: ping_type
-  type: STRING
-- mode: NULLABLE
-  name: os
-  type: STRING
-- mode: NULLABLE
-  name: app_version
-  type: INTEGER
-- mode: NULLABLE
-  name: app_build_id
-  type: STRING
-- mode: NULLABLE
-  name: channel
-  type: STRING
-- mode: NULLABLE
-  name: metric
-  type: STRING
-- mode: NULLABLE
-  name: metric_type
-  type: STRING
-- mode: NULLABLE
-  name: key
-  type: STRING
-- mode: NULLABLE
-  name: client_agg_type
-  type: STRING
-- mode: NULLABLE
-  name: agg_type
-  type: STRING
-- mode: NULLABLE
-  name: total_users
-  type: INTEGER
-- fields:
-  - mode: NULLABLE
-    name: key
-    type: STRING
-  - mode: NULLABLE
-    name: value
-    type: FLOAT
-  mode: REPEATED
-  name: aggregates
-  type: RECORD

sql/glam-fenix-dev/glam_etl/org_mozilla_fenix_glam_nightly__scalar_percentiles_v1/metadata.yaml

@@ -1,6 +0,0 @@
-friendly_name: |-
-  Glam Etl - Org Mozilla Fenix Glam Nightly - Scalar Percentiles
-description: |-
-  [DESCRIPTION_MISSING]
-owners:
-- efilho@mozilla.com

sql/glam-fenix-dev/glam_etl/org_mozilla_fenix_glam_nightly__scalar_percentiles_v1/query.sql

@@ -1,78 +0,0 @@
--- query for org_mozilla_fenix_glam_nightly__scalar_percentiles_v1;
-WITH flat_clients_scalar_aggregates AS (
-  SELECT
-    * EXCEPT (scalar_aggregates)
-  FROM
-    `glam-fenix-dev.glam_etl.org_mozilla_fenix_glam_nightly__clients_scalar_aggregates_v1`
-  CROSS JOIN
-    UNNEST(scalar_aggregates)
-),
--- Cross join with the attribute combinations to reduce the query complexity
--- with respect to the number of operations. A table with n rows cross joined
--- with a combination of m attributes will generate a new table with n*m rows.
--- The glob ("*") symbol can be understood as selecting all of values belonging
--- to that group.
-static_combos AS (
-  SELECT
-    combos.*
-  FROM
-    UNNEST(
-      ARRAY<STRUCT<ping_type STRING, os STRING, app_build_id STRING>>[
-        (NULL, NULL, NULL),
-        (NULL, NULL, "*"),
-        (NULL, "*", NULL),
-        ("*", NULL, NULL),
-        (NULL, "*", "*"),
-        ("*", NULL, "*"),
-        ("*", "*", NULL),
-        ("*", "*", "*")
-      ]
-    ) AS combos
-),
-all_combos AS (
-  SELECT
-    table.* EXCEPT (ping_type, os, app_build_id),
-    COALESCE(combo.ping_type, table.ping_type) AS ping_type,
-    COALESCE(combo.os, table.os) AS os,
-    COALESCE(combo.app_build_id, table.app_build_id) AS app_build_id
-  FROM
-    flat_clients_scalar_aggregates table
-  CROSS JOIN
-    static_combos combo
-),
-percentiles AS (
-  SELECT
-    ping_type,
-    os,
-    app_version,
-    app_build_id,
-    channel,
-    metric,
-    metric_type,
-    key,
-    agg_type AS client_agg_type,
-    'percentiles' AS agg_type,
-    APPROX_COUNT_DISTINCT(client_id) AS total_users,
-    APPROX_QUANTILES(value, 1000) AS aggregates
-  FROM
-    all_combos
-  GROUP BY
-    ping_type,
-    os,
-    app_version,
-    app_build_id,
-    channel,
-    metric,
-    metric_type,
-    key,
-    client_agg_type
-)
-SELECT
-  * REPLACE (
-    mozfun.glam.map_from_array_offsets_precise(
-      [0.1, 1.0, 5.0, 25.0, 50.0, 75.0, 95.0, 99.0, 99.9],
-      aggregates
-    ) AS aggregates
-  )
-FROM
-  percentiles

sql/glam-fenix-dev/glam_etl/org_mozilla_fenix_glam_nightly__scalar_percentiles_v1/schema.yaml

@@ -1,44 +0,0 @@
-fields:
-- mode: NULLABLE
-  name: ping_type
-  type: STRING
-- mode: NULLABLE
-  name: os
-  type: STRING
-- mode: NULLABLE
-  name: app_version
-  type: INTEGER
-- mode: NULLABLE
-  name: app_build_id
-  type: STRING
-- mode: NULLABLE
-  name: channel
-  type: STRING
-- mode: NULLABLE
-  name: metric
-  type: STRING
-- mode: NULLABLE
-  name: metric_type
-  type: STRING
-- mode: NULLABLE
-  name: key
-  type: STRING
-- mode: NULLABLE
-  name: client_agg_type
-  type: STRING
-- mode: NULLABLE
-  name: agg_type
-  type: STRING
-- mode: NULLABLE
-  name: total_users
-  type: INTEGER
-- fields:
-  - mode: NULLABLE
-    name: key
-    type: STRING
-  - mode: NULLABLE
-    name: value
-    type: FLOAT
-  mode: REPEATED
-  name: aggregates
-  type: RECORD

sql/glam-fenix-dev/glam_etl/org_mozilla_fenix_glam_nightly__view_probe_counts_v1/view.sql

@@ -12,16 +12,6 @@ WITH all_counts AS (
     *
   FROM
     `glam-fenix-dev.glam_etl.org_mozilla_fenix_glam_nightly__histogram_probe_counts_v1`
-  UNION ALL
-  SELECT
-    *
-  FROM
-    `glam-fenix-dev.glam_etl.org_mozilla_fenix_glam_nightly__scalar_percentiles_v1`
-  UNION ALL
-  SELECT
-    *
-  FROM
-    `glam-fenix-dev.glam_etl.org_mozilla_fenix_glam_nightly__histogram_percentiles_v1`
 )
 SELECT
   *

sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_desktop_beta_aggregates_v1/script.sql

@@ -43,17 +43,17 @@ THEN
       S.metric_type,
       S.total_users,
       S.histogram,
-      S.percentiles,
+      NULL,
       S.total_sample,
       S.non_norm_histogram,
-      S.non_norm_percentiles
+      NULL
     )
   WHEN MATCHED
 THEN
   UPDATE
     SET T.total_users = S.total_users,
     T.histogram = S.histogram,
-    T.percentiles = S.percentiles,
+    T.percentiles = NULL,
     T.total_sample = S.total_sample,
     T.non_norm_histogram = S.non_norm_histogram,
-    T.non_norm_percentiles = S.non_norm_percentiles
+    T.non_norm_percentiles = NULL

sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_desktop_nightly_aggregates_v1/script.sql

@@ -48,17 +48,17 @@ THEN
       S.metric_type,
       S.total_users,
       S.histogram,
-      S.percentiles,
+      NULL,
       S.total_sample,
       S.non_norm_histogram,
-      S.non_norm_percentiles
+      NULL
     )
   WHEN MATCHED
 THEN
   UPDATE
     SET T.total_users = S.total_users,
     T.histogram = S.histogram,
-    T.percentiles = S.percentiles,
+    T.percentiles = NULL,
     T.total_sample = S.total_sample,
     T.non_norm_histogram = S.non_norm_histogram,
-    T.non_norm_percentiles = S.non_norm_percentiles
+    T.non_norm_percentiles = NULL

sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_desktop_release_aggregates_v1/script.sql

@@ -48,17 +48,17 @@ THEN
       S.metric_type,
       S.total_users,
       S.histogram,
-      S.percentiles,
+      NULL,
       S.total_sample,
       S.non_norm_histogram,
-      S.non_norm_percentiles
+      NULL
     )
   WHEN MATCHED
 THEN
   UPDATE
     SET T.total_users = S.total_users,
     T.histogram = S.histogram,
-    T.percentiles = S.percentiles,
+    T.percentiles = NULL,
     T.total_sample = S.total_sample,
     T.non_norm_histogram = S.non_norm_histogram,
-    T.non_norm_percentiles = S.non_norm_percentiles
+    T.non_norm_percentiles = NULL

sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fenix_beta_aggregates_v1/script.sql

@@ -50,9 +50,9 @@ THEN
       S.client_agg_type,
       S.total_users,
       S.histogram,
-      S.percentiles,
+      NULL,
       S.non_norm_histogram,
-      S.non_norm_percentiles,
+      NULL,
       S.total_sample
     )
   WHEN MATCHED
@@ -60,7 +60,7 @@ THEN
   UPDATE
     SET T.total_users = S.total_users,
     T.histogram = S.histogram,
-    T.percentiles = S.percentiles,
+    T.percentiles = NULL,
     T.non_norm_histogram = S.non_norm_histogram,
-    T.non_norm_percentiles = S.non_norm_percentiles,
+    T.non_norm_percentiles = NULL,
     T.total_sample = S.total_sample

sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fenix_nightly_aggregates_v1/script.sql

@@ -50,9 +50,9 @@ THEN
       S.client_agg_type,
       S.total_users,
       S.histogram,
-      S.percentiles,
+      NULL,
       S.non_norm_histogram,
-      S.non_norm_percentiles,
+      NULL,
       S.total_sample
     )
   WHEN MATCHED
@@ -60,7 +60,7 @@ THEN
   UPDATE
     SET T.total_users = S.total_users,
     T.histogram = S.histogram,
-    T.percentiles = S.percentiles,
+    T.percentiles = NULL,
     T.non_norm_histogram = S.non_norm_histogram,
-    T.non_norm_percentiles = S.non_norm_percentiles,
+    T.non_norm_percentiles = NULL,
     T.total_sample = S.total_sample

sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fenix_release_aggregates_v1/script.sql

@@ -50,9 +50,9 @@ THEN
       S.client_agg_type,
       S.total_users,
       S.histogram,
-      S.percentiles,
+      NULL,
       S.non_norm_histogram,
-      S.non_norm_percentiles,
+      NULL,
       S.total_sample
     )
   WHEN MATCHED
@@ -60,7 +60,7 @@ THEN
   UPDATE
     SET T.total_users = S.total_users,
     T.histogram = S.histogram,
-    T.percentiles = S.percentiles,
+    T.percentiles = NULL,
     T.non_norm_histogram = S.non_norm_histogram,
-    T.non_norm_percentiles = S.non_norm_percentiles,
+    T.non_norm_percentiles = NULL,
     T.total_sample = S.total_sample

sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fog_beta_aggregates_v1/script.sql

@@ -50,9 +50,9 @@ THEN
       S.client_agg_type,
       S.total_users,
       S.histogram,
-      S.percentiles,
+      NULL,
       S.non_norm_histogram,
-      S.non_norm_percentiles,
+      NULL,
       S.total_sample
     )
   WHEN MATCHED
@@ -60,7 +60,7 @@ THEN
   UPDATE
     SET T.total_users = S.total_users,
     T.histogram = S.histogram,
-    T.percentiles = S.percentiles,
+    T.percentiles = NULL,
     T.non_norm_histogram = S.non_norm_histogram,
-    T.non_norm_percentiles = S.non_norm_percentiles,
+    T.non_norm_percentiles = NULL,
     T.total_sample = S.total_sample

sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fog_nightly_aggregates_v1/script.sql

@@ -50,9 +50,9 @@ THEN
       S.client_agg_type,
       S.total_users,
       S.histogram,
-      S.percentiles,
+      NULL,
       S.non_norm_histogram,
-      S.non_norm_percentiles,
+      NULL,
       S.total_sample
     )
   WHEN MATCHED
@@ -60,7 +60,7 @@ THEN
   UPDATE
     SET T.total_users = S.total_users,
     T.histogram = S.histogram,
-    T.percentiles = S.percentiles,
+    T.percentiles = NULL,
     T.non_norm_histogram = S.non_norm_histogram,
-    T.non_norm_percentiles = S.non_norm_percentiles,
+    T.non_norm_percentiles = NULL,
     T.total_sample = S.total_sample

sql/moz-fx-data-glam-prod-fca7/glam_etl/glam_fog_release_aggregates_v1/script.sql

@@ -50,9 +50,9 @@ THEN
       S.client_agg_type,
       S.total_users,
       S.histogram,
-      S.percentiles,
+      NULL,
       S.non_norm_histogram,
-      S.non_norm_percentiles,
+      NULL,
       S.total_sample
     )
   WHEN MATCHED
@@ -60,7 +60,7 @@ THEN
   UPDATE
     SET T.total_users = S.total_users,
     T.histogram = S.histogram,
-    T.percentiles = S.percentiles,
+    T.percentiles = NULL,
     T.non_norm_histogram = S.non_norm_histogram,
-    T.non_norm_percentiles = S.non_norm_percentiles,
+    T.non_norm_percentiles = NULL,
     T.total_sample = S.total_sample

sql/moz-fx-data-shared-prod/telemetry/client_probe_counts/view.sql

@@ -11,16 +11,6 @@ WITH all_counts AS (
     *
   FROM
     `moz-fx-data-shared-prod.telemetry_derived.clients_histogram_probe_counts_v1`
-  UNION ALL
-  SELECT
-    *
-  FROM
-    `moz-fx-data-shared-prod.telemetry_derived.scalar_percentiles_v1`
-  UNION ALL
-  SELECT
-    *
-  FROM
-    `moz-fx-data-shared-prod.telemetry_derived.histogram_percentiles_v1`
 )
 SELECT
   *

sql/moz-fx-data-shared-prod/telemetry_derived/glam_client_probe_counts_extract_v1/query.sql

@@ -13,13 +13,7 @@ WITH final_probe_extract AS (
     MAX(IF(agg_type = "histogram", mozfun.glam.histogram_cast_json(aggregates), NULL)) AS histogram,
     MAX(
       IF(agg_type = "histogram", mozfun.glam.histogram_cast_json(non_norm_aggregates), NULL)
-    ) AS non_norm_histogram,
-    MAX(
-      IF(agg_type = "percentiles", mozfun.glam.histogram_cast_json(aggregates), NULL)
-    ) AS percentiles,
-    MAX(
-      IF(agg_type = "percentiles", mozfun.glam.histogram_cast_json(non_norm_aggregates), NULL)
-    ) AS non_norm_percentiles
+    ) AS non_norm_histogram
   FROM
     `moz-fx-data-shared-prod.telemetry_derived.client_probe_counts`
   WHERE
@@ -79,14 +73,12 @@ SELECT
   cp.metric_type,
   total_users,
   histogram,
-  percentiles,
   CASE
     WHEN client_agg_type = ''
       THEN 0
     ELSE CAST(total_sample AS BIGNUMERIC)
   END AS total_sample,
-  non_norm_histogram,
-  non_norm_percentiles
+  non_norm_histogram
 FROM
   final_probe_extract cp
 LEFT JOIN
@@ -111,6 +103,4 @@ GROUP BY
   total_users,
   total_sample,
   histogram,
-  non_norm_histogram,
-  percentiles,
-  non_norm_percentiles
+  non_norm_histogram

sql/moz-fx-data-shared-prod/telemetry_derived/histogram_percentiles_v1/metadata.yaml

@@ -1,6 +0,0 @@
-friendly_name: |-
-  Telemetry Derived - Histogram Percentiles
-description: |-
-  [DESCRIPTION_MISSING]
-owners:
-- data-platform-infra-wg@mozilla.com

sql/moz-fx-data-shared-prod/telemetry_derived/histogram_percentiles_v1/query.sql

@@ -1,26 +0,0 @@
-SELECT
-  * EXCEPT (aggregates, non_norm_aggregates) REPLACE('percentiles' AS agg_type),
-  ARRAY<STRUCT<KEY STRING, value FLOAT64>>[
-    ('0.1', mozfun.glam.percentile(0.1, aggregates, metric_type)),
-    ('1', mozfun.glam.percentile(1, aggregates, metric_type)),
-    ('5', mozfun.glam.percentile(5, aggregates, metric_type)),
-    ('25', mozfun.glam.percentile(25, aggregates, metric_type)),
-    ('50', mozfun.glam.percentile(50, aggregates, metric_type)),
-    ('75', mozfun.glam.percentile(75, aggregates, metric_type)),
-    ('95', mozfun.glam.percentile(95, aggregates, metric_type)),
-    ('99', mozfun.glam.percentile(99, aggregates, metric_type)),
-    ('99.9', mozfun.glam.percentile(99.9, aggregates, metric_type))
-  ] AS aggregates,
-  ARRAY<STRUCT<KEY STRING, value FLOAT64>>[
-    ('0.1', mozfun.glam.percentile(0.1, non_norm_aggregates, metric_type)),
-    ('1', mozfun.glam.percentile(1, non_norm_aggregates, metric_type)),
-    ('5', mozfun.glam.percentile(5, non_norm_aggregates, metric_type)),
-    ('25', mozfun.glam.percentile(25, non_norm_aggregates, metric_type)),
-    ('50', mozfun.glam.percentile(50, non_norm_aggregates, metric_type)),
-    ('75', mozfun.glam.percentile(75, non_norm_aggregates, metric_type)),
-    ('95', mozfun.glam.percentile(95, non_norm_aggregates, metric_type)),
-    ('99', mozfun.glam.percentile(99, non_norm_aggregates, metric_type)),
-    ('99.9', mozfun.glam.percentile(99.9, non_norm_aggregates, metric_type))
-  ] AS non_norm_aggregates
-FROM
-  `moz-fx-data-shared-prod.telemetry_derived.clients_histogram_probe_counts_v1`

sql/moz-fx-data-shared-prod/telemetry_derived/histogram_percentiles_v1/schema.yaml

@@ -1,79 +0,0 @@
-fields:
-- mode: NULLABLE
-  name: app_version
-  type: INTEGER
-
-- mode: NULLABLE
-  name: app_build_id
-  type: STRING
-
-- mode: NULLABLE
-  name: channel
-  type: STRING
-
-- mode: NULLABLE
-  name: metric
-  type: STRING
-
-- mode: NULLABLE
-  name: metric_type
-  type: STRING
-
-- mode: NULLABLE
-  name: key
-  type: STRING
-
-- mode: NULLABLE
-  name: process
-  type: STRING
-
-
-- mode: NULLABLE
-  name: first_bucket
-  type: INTEGER
-
-- mode: NULLABLE
-  name: last_bucket
-  type: INTEGER
-
-- mode: NULLABLE
-  name: num_buckets
-  type: INTEGER
-
-- mode: NULLABLE
-  name: client_agg_type
-  type: STRING
-
-- mode: NULLABLE
-  name: agg_type
-  type: STRING
-
-- mode: NULLABLE
-  name: total_users
-  type: INTEGER
-
-- fields:
-  - mode: NULLABLE
-    name: KEY
-    type: STRING
-  - mode: NULLABLE
-    name: value
-    type: FLOAT
-  mode: REPEATED
-  name: aggregates
-  type: RECORD
-
-- fields:
-  - mode: NULLABLE
-    name: KEY
-    type: STRING
-  - mode: NULLABLE
-    name: value
-    type: FLOAT
-  mode: REPEATED
-  name: non_norm_aggregates
-  type: RECORD
-
-- name: os
-  type: STRING
-  mode: NULLABLE

sql/moz-fx-data-shared-prod/telemetry_derived/scalar_percentiles_v1/metadata.yaml

@@ -1,6 +0,0 @@
-friendly_name: |-
-  Telemetry Derived - Scalar Percentiles
-description: |-
-  [DESCRIPTION_MISSING]
-owners:
-- data-platform-infra-wg@mozilla.com

sql/moz-fx-data-shared-prod/telemetry_derived/scalar_percentiles_v1/query.sql

@@ -1,107 +0,0 @@
-WITH flat_clients_scalar_aggregates AS (
-  SELECT
-    *,
-    os = 'Windows'
-    AND channel = 'release' AS sampled,
-  FROM
-    `moz-fx-data-shared-prod.telemetry_derived.clients_scalar_aggregates_v1`
-  WHERE
-    submission_date = @submission_date
-    AND (@app_version IS NULL OR app_version = @app_version)
-),
-static_combos AS (
-  SELECT
-    NULL AS os,
-    NULL AS app_build_id
-  UNION ALL
-  SELECT
-    NULL AS os,
-    '*' AS app_build_id
-  UNION ALL
-  SELECT
-    '*' AS os,
-    NULL AS app_build_id
-  UNION ALL
-  SELECT
-    '*' AS os,
-    '*' AS app_build_id
-),
-all_combos AS (
-  SELECT
-    * EXCEPT (os, app_build_id),
-    COALESCE(combos.os, flat_table.os) AS os,
-    COALESCE(combos.app_build_id, flat_table.app_build_id) AS app_build_id
-  FROM
-    flat_clients_scalar_aggregates flat_table
-  CROSS JOIN
-    static_combos combos
-),
-user_aggregates AS (
-  SELECT
-    client_id,
-    IF(os = '*', NULL, os) AS os,
-    app_version,
-    IF(app_build_id = '*', NULL, app_build_id) AS app_build_id,
-    channel,
-    IF(MAX(sampled), 10, 1) AS user_count,
-    `moz-fx-data-shared-prod`.udf.merge_scalar_user_data(
-      ARRAY_CONCAT_AGG(scalar_aggregates)
-    ) AS scalar_aggregates
-  FROM
-    all_combos
-  GROUP BY
-    client_id,
-    os,
-    app_version,
-    app_build_id,
-    channel
-),
-percentiles AS (
-  SELECT
-    os,
-    app_version,
-    app_build_id,
-    channel,
-    metric,
-    metric_type,
-    key,
-    process,
-    -- empty columns to match clients_histogram_probe_counts_v1 schema
-    NULL AS first_bucket,
-    NULL AS last_bucket,
-    NULL AS num_buckets,
-    agg_type AS client_agg_type,
-    'percentiles' AS agg_type,
-    SUM(user_count) AS total_users,
-    APPROX_QUANTILES(value, 1000) AS aggregates
-  FROM
-    user_aggregates
-  CROSS JOIN
-    UNNEST(scalar_aggregates)
-  GROUP BY
-    os,
-    app_version,
-    app_build_id,
-    channel,
-    metric,
-    metric_type,
-    key,
-    process,
-    client_agg_type
-),
-aggregated AS (
-  SELECT
-    * REPLACE (
-      mozfun.glam.map_from_array_offsets_precise(
-        [0.1, 1.0, 5.0, 25.0, 50.0, 75.0, 95.0, 99.0, 99.9],
-        aggregates
-      ) AS aggregates
-    )
-  FROM
-    percentiles
-)
-SELECT
-  *,
-  aggregates AS non_norm_aggregates
-FROM
-  aggregated

sql/moz-fx-data-shared-prod/telemetry_derived/scalar_percentiles_v1/schema.yaml

@@ -1,78 +0,0 @@
-fields:
-- mode: NULLABLE
-  name: os
-  type: STRING
-
-- mode: NULLABLE
-  name: app_version
-  type: INTEGER
-
-- mode: NULLABLE
-  name: app_build_id
-  type: STRING
-
-- mode: NULLABLE
-  name: channel
-  type: STRING
-
-- mode: NULLABLE
-  name: metric
-  type: STRING
-
-- mode: NULLABLE
-  name: metric_type
-  type: STRING
-
-- mode: NULLABLE
-  name: key
-  type: STRING
-
-- mode: NULLABLE
-  name: process
-  type: STRING
-
-- mode: NULLABLE
-  name: first_bucket
-  type: INTEGER
-
-- mode: NULLABLE
-  name: last_bucket
-  type: INTEGER
-
-- mode: NULLABLE
-  name: num_buckets
-  type: INTEGER
-
-- mode: NULLABLE
-  name: client_agg_type
-  type: STRING
-
-- mode: NULLABLE
-  name: agg_type
-  type: STRING
-
-- mode: NULLABLE
-  name: total_users
-  type: INTEGER
-
-- fields:
-  - mode: NULLABLE
-    name: key
-    type: STRING
-  - mode: NULLABLE
-    name: value
-    type: FLOAT
-  mode: REPEATED
-  name: aggregates
-  type: RECORD
-
-- fields:
-  - mode: NULLABLE
-    name: key
-    type: STRING
-  - mode: NULLABLE
-    name: value
-    type: FLOAT
-  mode: REPEATED
-  name: non_norm_aggregates
-  type: RECORD

sql/mozfun/glam/histogram_cast_struct/udf.sql

@@ -13,7 +13,7 @@ AS
   const json_dict = JSON.parse(json_str);
   const entries = Object.entries(json_dict).map(
       (r)=>Object.fromEntries(
-        [["KEY", parseFloat(r[0])],["value", parseFloat(r[1])]]
+        [["KEY", r[0]],["value", parseFloat(r[1])]]
       )
   );
   return entries;
@@ -28,4 +28,8 @@ SELECT
     ARRAY<STRUCT<key STRING, value FLOAT64>>[("0", 0.1111), ("1", 0.6667), ("2", 0), ("10", 100)],
     glam.histogram_cast_struct('{"0":0.1111,"1":0.6667,"2":0,"10":100}')
   ),
-  assert.array_empty(glam.histogram_cast_struct('{}'))
+  assert.array_empty(glam.histogram_cast_struct('{}')),
+  assert.array_equals(
+    ARRAY<STRUCT<key STRING, value FLOAT64>>[("always", 0.5), ("never", 0.5)],
+    glam.histogram_cast_struct('{"always":0.5,"never":0.5}')
+  )

tests/sql/glam-fenix-dev/glam_etl/org_mozilla_fenix_glam_nightly__extract_probe_counts_v1/glam-fenix-dev.glam_etl.org_mozilla_fenix_glam_nightly__view_probe_counts_v1.schema.yaml

@@ -1,43 +0,0 @@
-- mode: NULLABLE
-  name: ping_type
-  type: STRING
-- mode: NULLABLE
-  name: os
-  type: STRING
-- mode: NULLABLE
-  name: app_version
-  type: INTEGER
-- mode: NULLABLE
-  name: app_build_id
-  type: STRING
-- mode: NULLABLE
-  name: channel
-  type: STRING
-- mode: NULLABLE
-  name: metric
-  type: STRING
-- mode: NULLABLE
-  name: metric_type
-  type: STRING
-- mode: NULLABLE
-  name: key
-  type: STRING
-- mode: NULLABLE
-  name: client_agg_type
-  type: STRING
-- mode: NULLABLE
-  name: agg_type
-  type: STRING
-- mode: NULLABLE
-  name: total_users
-  type: INTEGER
-- fields:
-  - mode: NULLABLE
-    name: key
-    type: STRING
-  - mode: NULLABLE
-    name: value
-    type: FLOAT
-  mode: REPEATED
-  name: aggregates
-  type: RECORD

tests/sql/glam-fenix-dev/glam_etl/org_mozilla_fenix_glam_nightly__extract_probe_counts_v1/test_minimal/data.py

@@ -1,61 +0,0 @@
-"""Testing data for query."""
-
-from pathlib import Path
-
-import yaml
-
-ROOT = Path(__file__).parent
-
-SUBMISSION_DATE = "2020-10-01"
-APP_BUILD_ID = "2020100100"
-
-SCALAR_PERCENTILES = [
-    {
-        "agg_type": "percentiles",
-        "aggregates": [
-            {"key": "5", "value": 4.0},
-            {"key": "25", "value": 4.0},
-            {"key": "50", "value": 4.0},
-            {"key": "75", "value": 8.0},
-            {"key": "95", "value": 8.0},
-        ],
-        "app_build_id": "*",
-        "app_version": 84,
-        "channel": "*",
-        "client_agg_type": "count",
-        "key": "",
-        "metric": "places_manager_write_query_count",
-        "metric_type": "counter",
-        "os": "*",
-        "ping_type": "*",
-        "total_users": 16,
-    }
-]
-
-EXPECT = [
-    {
-        "build_id": "*",
-        "channel": "*",
-        "client_agg_type": "count",
-        "metric": "places_manager_write_query_count",
-        "metric_key": "",
-        "metric_type": "counter",
-        "os": "*",
-        "percentiles": '{"5":4,"25":4,"50":4,"75":8,"95":8}',
-        "ping_type": "*",
-        "total_users": 16,
-        "version": 84,
-    }
-]
-
-prefix = "glam_etl"
-tables = [
-    (
-        f"{prefix}.org_mozilla_fenix_glam_nightly__view_probe_counts_v1.yaml",
-        SCALAR_PERCENTILES,
-    ),
-    ("expect.yaml", EXPECT),
-]
-for name, data in tables:
-    with (ROOT / name).open("w") as fp:
-        yaml.dump(data, fp)

tests/sql/glam-fenix-dev/glam_etl/org_mozilla_fenix_glam_nightly__extract_probe_counts_v1/test_minimal/expect.yaml

@@ -1,11 +0,0 @@
-- build_id: '*'
-  channel: '*'
-  client_agg_type: count
-  metric: places_manager_write_query_count
-  metric_key: ''
-  metric_type: counter
-  os: '*'
-  percentiles: '{"5":4,"25":4,"50":4,"75":8,"95":8}'
-  ping_type: '*'
-  total_users: 16
-  version: 84

tests/sql/glam-fenix-dev/glam_etl/org_mozilla_fenix_glam_nightly__extract_probe_counts_v1/test_minimal/glam-fenix-dev.glam_etl.org_mozilla_fenix_glam_nightly__view_probe_counts_v1.yaml

@@ -1,22 +0,0 @@
-- agg_type: percentiles
-  aggregates:
-  - key: '5'
-    value: 4.0
-  - key: '25'
-    value: 4.0
-  - key: '50'
-    value: 4.0
-  - key: '75'
-    value: 8.0
-  - key: '95'
-    value: 8.0
-  app_build_id: '*'
-  app_version: 84
-  channel: '*'
-  client_agg_type: count
-  key: ''
-  metric: places_manager_write_query_count
-  metric_type: counter
-  os: '*'
-  ping_type: '*'
-  total_users: 16

tests/sql/glam-fenix-dev/glam_etl/org_mozilla_fenix_glam_nightly__extract_probe_counts_v1/test_minimal/query_params.yaml

@@ -1,12 +0,0 @@
-- name: submission_date
-  type: DATE
-  value: '2020-10-01'
-- name: min_sample_id
-  type: INT64
-  value: 0
-- name: max_sample_id
-  type: INT64
-  value: 99
-- name: sample_size
-  type: INT64
-  value: 100

tests/sql/glam-fenix-dev/glam_etl/org_mozilla_fenix_glam_nightly__histogram_percentiles_v1/glam-fenix-dev.glam_etl.org_mozilla_fenix_glam_nightly__histogram_probe_counts_v1.schema.yaml

@@ -1,43 +0,0 @@
-- mode: NULLABLE
-  name: ping_type
-  type: STRING
-- mode: NULLABLE
-  name: os
-  type: STRING
-- mode: NULLABLE
-  name: app_version
-  type: INTEGER
-- mode: NULLABLE
-  name: app_build_id
-  type: STRING
-- mode: NULLABLE
-  name: channel
-  type: STRING
-- mode: NULLABLE
-  name: metric
-  type: STRING
-- mode: NULLABLE
-  name: metric_type
-  type: STRING
-- mode: NULLABLE
-  name: key
-  type: STRING
-- mode: NULLABLE
-  name: client_agg_type
-  type: STRING
-- mode: NULLABLE
-  name: agg_type
-  type: STRING
-- mode: NULLABLE
-  name: total_users
-  type: INTEGER
-- fields:
-  - mode: NULLABLE
-    name: key
-    type: STRING
-  - mode: NULLABLE
-    name: value
-    type: FLOAT
-  mode: REPEATED
-  name: aggregates
-  type: RECORD

tests/sql/glam-fenix-dev/glam_etl/org_mozilla_fenix_glam_nightly__histogram_percentiles_v1/test_minimal/data.py

@@ -1,66 +0,0 @@
-"""Testing data for query."""
-
-from pathlib import Path
-
-import yaml
-
-ROOT = Path(__file__).parent
-
-SUBMISSION_DATE = "2020-10-01"
-APP_BUILD_ID = "2020100100"
-
-HISTOGRAM_PROBE_COUNTS = [
-    {
-        "agg_type": "histogram",
-        "aggregates": [
-            # 101 buckets with uniform proportion, 0-100 inclusive.
-            {"key": str(key), "value": 1 / 101}
-            for key in range(101)
-        ],
-        "app_build_id": "*",
-        "app_version": 84,
-        "channel": "*",
-        "client_agg_type": "summed_histogram",
-        "key": "",
-        "metric": "network_tcp_connection",
-        "metric_type": "timing_distribution",
-        "os": "*",
-        "ping_type": "*",
-        "total_users": 1,
-    }
-]
-
-EXPECT = [
-    {
-        "agg_type": "percentiles",
-        "aggregates": [
-            {"key": "5", "value": 5.0},
-            {"key": "25", "value": 25.0},
-            {"key": "50", "value": 50.0},
-            {"key": "75", "value": 75.0},
-            {"key": "95", "value": 95.0},
-        ],
-        "app_build_id": "*",
-        "app_version": 84,
-        "channel": "*",
-        "client_agg_type": "summed_histogram",
-        "key": "",
-        "metric": "network_tcp_connection",
-        "metric_type": "timing_distribution",
-        "os": "*",
-        "ping_type": "*",
-        "total_users": 1,
-    }
-]
-
-prefix = "glam_etl"
-tables = [
-    (
-        f"{prefix}.org_mozilla_fenix_glam_nightly__histogram_probe_counts_v1.yaml",
-        HISTOGRAM_PROBE_COUNTS,
-    ),
-    ("expect.yaml", EXPECT),
-]
-for name, data in tables:
-    with (ROOT / name).open("w") as fp:
-        yaml.dump(data, fp)

tests/sql/glam-fenix-dev/glam_etl/org_mozilla_fenix_glam_nightly__histogram_percentiles_v1/test_minimal/expect.yaml

@@ -1,22 +0,0 @@
-- agg_type: percentiles
-  aggregates:
-  - key: '5'
-    value: 5.0
-  - key: '25'
-    value: 25.0
-  - key: '50'
-    value: 50.0
-  - key: '75'
-    value: 75.0
-  - key: '95'
-    value: 95.0
-  app_build_id: '*'
-  app_version: 84
-  channel: '*'
-  client_agg_type: summed_histogram
-  key: ''
-  metric: network_tcp_connection
-  metric_type: timing_distribution
-  os: '*'
-  ping_type: '*'
-  total_users: 1

tests/sql/glam-fenix-dev/glam_etl/org_mozilla_fenix_glam_nightly__histogram_percentiles_v1/test_minimal/glam-fenix-dev.glam_etl.org_mozilla_fenix_glam_nightly__histogram_probe_counts_v1.yaml

@@ -1,214 +0,0 @@
-- agg_type: histogram
-  aggregates:
-  - key: '0'
-    value: 0.009900990099009901
-  - key: '1'
-    value: 0.009900990099009901
-  - key: '2'
-    value: 0.009900990099009901
-  - key: '3'
-    value: 0.009900990099009901
-  - key: '4'
-    value: 0.009900990099009901
-  - key: '5'
-    value: 0.009900990099009901
-  - key: '6'
-    value: 0.009900990099009901
-  - key: '7'
-    value: 0.009900990099009901
-  - key: '8'
-    value: 0.009900990099009901
-  - key: '9'
-    value: 0.009900990099009901
-  - key: '10'
-    value: 0.009900990099009901
-  - key: '11'
-    value: 0.009900990099009901
-  - key: '12'
-    value: 0.009900990099009901
-  - key: '13'
-    value: 0.009900990099009901
-  - key: '14'
-    value: 0.009900990099009901
-  - key: '15'
-    value: 0.009900990099009901
-  - key: '16'
-    value: 0.009900990099009901
-  - key: '17'
-    value: 0.009900990099009901
-  - key: '18'
-    value: 0.009900990099009901
-  - key: '19'
-    value: 0.009900990099009901
-  - key: '20'
-    value: 0.009900990099009901
-  - key: '21'
-    value: 0.009900990099009901
-  - key: '22'
-    value: 0.009900990099009901
-  - key: '23'
-    value: 0.009900990099009901
-  - key: '24'
-    value: 0.009900990099009901
-  - key: '25'
-    value: 0.009900990099009901
-  - key: '26'
-    value: 0.009900990099009901
-  - key: '27'
-    value: 0.009900990099009901
-  - key: '28'
-    value: 0.009900990099009901
-  - key: '29'
-    value: 0.009900990099009901
-  - key: '30'
-    value: 0.009900990099009901
-  - key: '31'
-    value: 0.009900990099009901
-  - key: '32'
-    value: 0.009900990099009901
-  - key: '33'
-    value: 0.009900990099009901
-  - key: '34'
-    value: 0.009900990099009901
-  - key: '35'
-    value: 0.009900990099009901
-  - key: '36'
-    value: 0.009900990099009901
-  - key: '37'
-    value: 0.009900990099009901
-  - key: '38'
-    value: 0.009900990099009901
-  - key: '39'
-    value: 0.009900990099009901
-  - key: '40'
-    value: 0.009900990099009901
-  - key: '41'
-    value: 0.009900990099009901
-  - key: '42'
-    value: 0.009900990099009901
-  - key: '43'
-    value: 0.009900990099009901
-  - key: '44'
-    value: 0.009900990099009901
-  - key: '45'
-    value: 0.009900990099009901
-  - key: '46'
-    value: 0.009900990099009901
-  - key: '47'
-    value: 0.009900990099009901
-  - key: '48'
-    value: 0.009900990099009901
-  - key: '49'
-    value: 0.009900990099009901
-  - key: '50'
-    value: 0.009900990099009901
-  - key: '51'
-    value: 0.009900990099009901
-  - key: '52'
-    value: 0.009900990099009901
-  - key: '53'
-    value: 0.009900990099009901
-  - key: '54'
-    value: 0.009900990099009901
-  - key: '55'
-    value: 0.009900990099009901
-  - key: '56'
-    value: 0.009900990099009901
-  - key: '57'
-    value: 0.009900990099009901
-  - key: '58'
-    value: 0.009900990099009901
-  - key: '59'
-    value: 0.009900990099009901
-  - key: '60'
-    value: 0.009900990099009901
-  - key: '61'
-    value: 0.009900990099009901
-  - key: '62'
-    value: 0.009900990099009901
-  - key: '63'
-    value: 0.009900990099009901
-  - key: '64'
-    value: 0.009900990099009901
-  - key: '65'
-    value: 0.009900990099009901
-  - key: '66'
-    value: 0.009900990099009901
-  - key: '67'
-    value: 0.009900990099009901
-  - key: '68'
-    value: 0.009900990099009901
-  - key: '69'
-    value: 0.009900990099009901
-  - key: '70'
-    value: 0.009900990099009901
-  - key: '71'
-    value: 0.009900990099009901
-  - key: '72'
-    value: 0.009900990099009901
-  - key: '73'
-    value: 0.009900990099009901
-  - key: '74'
-    value: 0.009900990099009901
-  - key: '75'
-    value: 0.009900990099009901
-  - key: '76'
-    value: 0.009900990099009901
-  - key: '77'
-    value: 0.009900990099009901
-  - key: '78'
-    value: 0.009900990099009901
-  - key: '79'
-    value: 0.009900990099009901
-  - key: '80'
-    value: 0.009900990099009901
-  - key: '81'
-    value: 0.009900990099009901
-  - key: '82'
-    value: 0.009900990099009901
-  - key: '83'
-    value: 0.009900990099009901
-  - key: '84'
-    value: 0.009900990099009901
-  - key: '85'
-    value: 0.009900990099009901
-  - key: '86'
-    value: 0.009900990099009901
-  - key: '87'
-    value: 0.009900990099009901
-  - key: '88'
-    value: 0.009900990099009901
-  - key: '89'
-    value: 0.009900990099009901
-  - key: '90'
-    value: 0.009900990099009901
-  - key: '91'
-    value: 0.009900990099009901
-  - key: '92'
-    value: 0.009900990099009901
-  - key: '93'
-    value: 0.009900990099009901
-  - key: '94'
-    value: 0.009900990099009901
-  - key: '95'
-    value: 0.009900990099009901
-  - key: '96'
-    value: 0.009900990099009901
-  - key: '97'
-    value: 0.009900990099009901
-  - key: '98'
-    value: 0.009900990099009901
-  - key: '99'
-    value: 0.009900990099009901
-  - key: '100'
-    value: 0.009900990099009901
-  app_build_id: '*'
-  app_version: 84
-  channel: '*'
-  client_agg_type: summed_histogram
-  key: ''
-  metric: network_tcp_connection
-  metric_type: timing_distribution
-  os: '*'
-  ping_type: '*'
-  total_users: 1

tests/sql/glam-fenix-dev/glam_etl/org_mozilla_fenix_glam_nightly__histogram_percentiles_v1/test_minimal/query_params.yaml

@@ -1,12 +0,0 @@
-- name: submission_date
-  type: DATE
-  value: '2020-10-01'
-- name: min_sample_id
-  type: INT64
-  value: 0
-- name: max_sample_id
-  type: INT64
-  value: 99
-- name: sample_size
-  type: INT64
-  value: 100

tests/sql/glam-fenix-dev/glam_etl/org_mozilla_fenix_glam_nightly__scalar_percentiles_v1/glam-fenix-dev.glam_etl.org_mozilla_fenix_glam_nightly__clients_scalar_aggregates_v1.schema.yaml

@@ -1,37 +0,0 @@
-- mode: NULLABLE
-  name: client_id
-  type: STRING
-- mode: NULLABLE
-  name: ping_type
-  type: STRING
-- mode: NULLABLE
-  name: os
-  type: STRING
-- mode: NULLABLE
-  name: app_version
-  type: INTEGER
-- mode: NULLABLE
-  name: app_build_id
-  type: STRING
-- mode: NULLABLE
-  name: channel
-  type: STRING
-- fields:
-  - mode: NULLABLE
-    name: metric
-    type: STRING
-  - mode: NULLABLE
-    name: metric_type
-    type: STRING
-  - mode: NULLABLE
-    name: key
-    type: STRING
-  - mode: NULLABLE
-    name: agg_type
-    type: STRING
-  - mode: NULLABLE
-    name: value
-    type: FLOAT
-  mode: REPEATED
-  name: scalar_aggregates
-  type: RECORD

tests/sql/glam-fenix-dev/glam_etl/org_mozilla_fenix_glam_nightly__scalar_percentiles_v1/test_minimal/data.py

@@ -1,84 +0,0 @@
-"""Testing data for query."""
-
-from pathlib import Path
-from uuid import uuid4
-
-import yaml
-
-ROOT = Path(__file__).parent
-
-SUBMISSION_DATE = "2020-10-01"
-APP_BUILD_ID = "2020100100"
-
-CLIENTS_SCALAR_AGGREGATES = [
-    {
-        "client_id": str(uuid4()),
-        "ping_type": "*",
-        "os": "*",
-        "app_version": 84,
-        "app_build_id": "*",
-        "channel": "*",
-        "scalar_aggregates": [
-            {
-                "metric": "places_manager_write_query_count",
-                "metric_type": "counter",
-                "key": "",
-                "agg_type": "count",
-                "value": 4.0,
-            }
-        ],
-    },
-    {
-        "client_id": str(uuid4()),
-        "ping_type": "*",
-        "os": "*",
-        "app_version": 84,
-        "app_build_id": "*",
-        "channel": "*",
-        "scalar_aggregates": [
-            {
-                "metric": "places_manager_write_query_count",
-                "metric_type": "counter",
-                "key": "",
-                "agg_type": "count",
-                "value": 8.0,
-            }
-        ],
-    },
-]
-
-# TODO: the total user count is very wrong
-EXPECT = [
-    {
-        "agg_type": "percentiles",
-        "aggregates": [
-            {"key": "5", "value": 4.0},
-            {"key": "25", "value": 4.0},
-            {"key": "50", "value": 4.0},
-            {"key": "75", "value": 8.0},
-            {"key": "95", "value": 8.0},
-        ],
-        "app_build_id": "*",
-        "app_version": 84,
-        "channel": "*",
-        "client_agg_type": "count",
-        "key": "",
-        "metric": "places_manager_write_query_count",
-        "metric_type": "counter",
-        "os": "*",
-        "ping_type": "*",
-        "total_users": 2,
-    }
-]
-
-prefix = "glam_etl"
-tables = [
-    (
-        f"{prefix}.org_mozilla_fenix_glam_nightly__clients_scalar_aggregates_v1.yaml",
-        CLIENTS_SCALAR_AGGREGATES,
-    ),
-    ("expect.yaml", EXPECT),
-]
-for name, data in tables:
-    with (ROOT / name).open("w") as fp:
-        yaml.dump(data, fp)

tests/sql/glam-fenix-dev/glam_etl/org_mozilla_fenix_glam_nightly__scalar_percentiles_v1/test_minimal/expect.yaml

@@ -1,30 +0,0 @@
-- agg_type: percentiles
-  aggregates:
-  - key: '0.1'
-    value: 4.0
-  - key: '1'
-    value: 4.0
-  - key: '5'
-    value: 4.0
-  - key: '25'
-    value: 4.0
-  - key: '50'
-    value: 4.0
-  - key: '75'
-    value: 8.0
-  - key: '95'
-    value: 8.0
-  - key: '99'
-    value: 8.0
-  - key: '99.9'
-    value: 8.0
-  app_build_id: '*'
-  app_version: 84
-  channel: '*'
-  client_agg_type: count
-  key: ''
-  metric: places_manager_write_query_count
-  metric_type: counter
-  os: '*'
-  ping_type: '*'
-  total_users: 2

tests/sql/glam-fenix-dev/glam_etl/org_mozilla_fenix_glam_nightly__scalar_percentiles_v1/test_minimal/glam-fenix-dev.glam_etl.org_mozilla_fenix_glam_nightly__clients_scalar_aggregates_v1.yaml

@@ -1,24 +0,0 @@
-- app_build_id: '*'
-  app_version: 84
-  channel: '*'
-  client_id: e321e7d8-4187-44b7-be82-803cd3819958
-  os: '*'
-  ping_type: '*'
-  scalar_aggregates:
-  - agg_type: count
-    key: ''
-    metric: places_manager_write_query_count
-    metric_type: counter
-    value: 4.0
-- app_build_id: '*'
-  app_version: 84
-  channel: '*'
-  client_id: fe8b857f-f7df-47b8-baca-ad4d44982f86
-  os: '*'
-  ping_type: '*'
-  scalar_aggregates:
-  - agg_type: count
-    key: ''
-    metric: places_manager_write_query_count
-    metric_type: counter
-    value: 8.0

tests/sql/glam-fenix-dev/glam_etl/org_mozilla_fenix_glam_nightly__scalar_percentiles_v1/test_minimal/query_params.yaml

@@ -1,12 +0,0 @@
-- name: submission_date
-  type: DATE
-  value: '2020-10-01'
-- name: min_sample_id
-  type: INT64
-  value: 0
-- name: max_sample_id
-  type: INT64
-  value: 99
-- name: sample_size
-  type: INT64
-  value: 100