remove latest_version column for histogram aggregates (#2270)

* remove latest_version column for histogram aggregates

* changed the run_glam_sql query based on comments

* remove latest_version dimension from histogram aggregates

script/glam/run_glam_sql

@@ -167,7 +167,7 @@ function run_desktop_sql {
         run_init "clients_histogram_aggregates_v1"
         run_query "clients_histogram_aggregates_new_v1"
         run_partitioned_query "clients_histogram_aggregates_v1" \
-            true "submission_date" "sample_id,app_version,channel"
+            1 true "submission_date" "sample_id,app_version,channel"
     fi
 
     run_partitioned_query "clients_histogram_bucket_counts_v1" 10 false
@@ -318,7 +318,7 @@ function main {
     if $reset; then
         bq rm -r -f "$DATASET"
     fi
-    if ! bq ls "${PROJECT}:${DATASET}" &> /dev/null; then
+    if ! bq ls "${DST_PROJECT}:${DATASET}" &> /dev/null; then
         bq mk "$DATASET"
     fi
 

sql/moz-fx-data-shared-prod/telemetry_derived/clients_histogram_aggregates_v1/init.sql

@@ -11,7 +11,6 @@ CREATE TABLE IF NOT EXISTS
       first_bucket INT64,
       last_bucket INT64,
       num_buckets INT64,
-      latest_version INT64,
       metric STRING,
       metric_type STRING,
       key STRING,

sql/moz-fx-data-shared-prod/telemetry_derived/clients_histogram_aggregates_v1/query.sql

@@ -3,7 +3,6 @@ CREATE TEMP FUNCTION udf_merged_user_data(old_aggs ANY TYPE, new_aggs ANY TYPE)
     first_bucket INT64,
     last_bucket INT64,
     num_buckets INT64,
-    latest_version INT64,
     metric STRING,
     metric_type STRING,
     key STRING,
@@ -25,7 +24,6 @@ CREATE TEMP FUNCTION udf_merged_user_data(old_aggs ANY TYPE, new_aggs ANY TYPE)
         first_bucket,
         last_bucket,
         num_buckets,
-        latest_version,
         metric,
         metric_type,
         key,
@@ -37,7 +35,6 @@ CREATE TEMP FUNCTION udf_merged_user_data(old_aggs ANY TYPE, new_aggs ANY TYPE)
         first_bucket,
         last_bucket,
         num_buckets,
-        latest_version,
         metric,
         metric_type,
         key,
@@ -48,7 +45,6 @@ CREATE TEMP FUNCTION udf_merged_user_data(old_aggs ANY TYPE, new_aggs ANY TYPE)
         first_bucket,
         last_bucket,
         num_buckets,
-        latest_version,
         metric,
         metric_type,
         key,
@@ -96,10 +92,22 @@ merged AS
     COALESCE(old_data.app_build_id, new_data.app_build_id) AS app_build_id,
     COALESCE(old_data.channel, new_data.channel) AS channel,
     old_data.histogram_aggregates AS old_aggs,
-    new_data.histogram_aggregates AS new_aggs
+    ARRAY(
+    SELECT AS STRUCT
+      first_bucket,
+      last_bucket,
+      num_buckets,
+      metric,
+      metric_type,
+      key,
+      process,
+      agg_type,
+      aggregates
+    FROM UNNEST(new_data.histogram_aggregates)
+    ) AS new_aggs
   FROM clients_histogram_aggregates_old AS old_data
-    FULL OUTER JOIN clients_histogram_aggregates_new AS new_data
-    ON new_data.join_key = old_data.join_key)
+  FULL OUTER JOIN clients_histogram_aggregates_new AS new_data
+  ON new_data.join_key = old_data.join_key)
 
 SELECT
   @submission_date AS submission_date,