From b5b10ec5bb95d79e4fde841b12037f5cc0f3f4fe Mon Sep 17 00:00:00 2001
From: Alekhya <88394696+alekhyamoz@users.noreply.github.com>
Date: Fri, 20 Aug 2021 14:52:57 -0400
Subject: [PATCH] remove latest_version column for histogram aggregates (#2270)

* remove latest_version column for histogram aggregates

* changed the run_glam_sql query based on comments

* remove latest_version dimension from histogram aggregates
---
 script/glam/run_glam_sql                      |  4 ++--
 .../clients_histogram_aggregates_v1/init.sql  |  1 -
 .../clients_histogram_aggregates_v1/query.sql | 22 +++++++++++++------
 3 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/script/glam/run_glam_sql b/script/glam/run_glam_sql
index 0d7353e36c..027b309a49 100755
--- a/script/glam/run_glam_sql
+++ b/script/glam/run_glam_sql
@@ -167,7 +167,7 @@ function run_desktop_sql {
         run_init "clients_histogram_aggregates_v1"
         run_query "clients_histogram_aggregates_new_v1"
         run_partitioned_query "clients_histogram_aggregates_v1" \
-            true "submission_date" "sample_id,app_version,channel"
+            1 true "submission_date" "sample_id,app_version,channel"
     fi
 
     run_partitioned_query "clients_histogram_bucket_counts_v1" 10 false
@@ -318,7 +318,7 @@ function main {
     if $reset; then
         bq rm -r -f "$DATASET"
     fi
-    if ! bq ls "${PROJECT}:${DATASET}" &> /dev/null; then
+    if ! bq ls "${DST_PROJECT}:${DATASET}" &> /dev/null; then
         bq mk "$DATASET"
     fi
 
diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/clients_histogram_aggregates_v1/init.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/clients_histogram_aggregates_v1/init.sql
index 105dc17b8f..db30d921e1 100755
--- a/sql/moz-fx-data-shared-prod/telemetry_derived/clients_histogram_aggregates_v1/init.sql
+++ b/sql/moz-fx-data-shared-prod/telemetry_derived/clients_histogram_aggregates_v1/init.sql
@@ -11,7 +11,6 @@ CREATE TABLE IF NOT EXISTS
       first_bucket INT64,
       last_bucket INT64,
       num_buckets INT64,
-      latest_version INT64,
       metric STRING,
       metric_type STRING,
       key STRING,
diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/clients_histogram_aggregates_v1/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/clients_histogram_aggregates_v1/query.sql
index 136ffb2018..db9df2e5a1 100755
--- a/sql/moz-fx-data-shared-prod/telemetry_derived/clients_histogram_aggregates_v1/query.sql
+++ b/sql/moz-fx-data-shared-prod/telemetry_derived/clients_histogram_aggregates_v1/query.sql
@@ -3,7 +3,6 @@ CREATE TEMP FUNCTION udf_merged_user_data(old_aggs ANY TYPE, new_aggs ANY TYPE)
     first_bucket INT64,
     last_bucket INT64,
     num_buckets INT64,
-    latest_version INT64,
     metric STRING,
     metric_type STRING,
     key STRING,
@@ -25,7 +24,6 @@ CREATE TEMP FUNCTION udf_merged_user_data(old_aggs ANY TYPE, new_aggs ANY TYPE)
         first_bucket,
         last_bucket,
         num_buckets,
-        latest_version,
         metric,
         metric_type,
         key,
@@ -37,7 +35,6 @@ CREATE TEMP FUNCTION udf_merged_user_data(old_aggs ANY TYPE, new_aggs ANY TYPE)
         first_bucket,
         last_bucket,
         num_buckets,
-        latest_version,
         metric,
         metric_type,
         key,
@@ -48,7 +45,6 @@ CREATE TEMP FUNCTION udf_merged_user_data(old_aggs ANY TYPE, new_aggs ANY TYPE)
         first_bucket,
         last_bucket,
         num_buckets,
-        latest_version,
         metric,
         metric_type,
         key,
@@ -96,10 +92,22 @@ merged AS
     COALESCE(old_data.app_build_id, new_data.app_build_id) AS app_build_id,
     COALESCE(old_data.channel, new_data.channel) AS channel,
     old_data.histogram_aggregates AS old_aggs,
-    new_data.histogram_aggregates AS new_aggs
+    ARRAY(
+    SELECT AS STRUCT
+      first_bucket,
+      last_bucket,
+      num_buckets,
+      metric,
+      metric_type,
+      key,
+      process,
+      agg_type,
+      aggregates
+    FROM UNNEST(new_data.histogram_aggregates)
+    ) AS new_aggs
   FROM clients_histogram_aggregates_old AS old_data
-    FULL OUTER JOIN clients_histogram_aggregates_new AS new_data
-    ON new_data.join_key = old_data.join_key)
+  FULL OUTER JOIN clients_histogram_aggregates_new AS new_data
+  ON new_data.join_key = old_data.join_key)
 
 SELECT
   @submission_date AS submission_date,