DENG-4481 Add profile_group_id to telemetry_derived.clients_first_see… (#6044)

2024-08-09 22:10:17 -05:00 · 2024-08-09 22:10:17 -05:00 · fb064e8f4f
--- a/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/query.sql
+++ b/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/query.sql
@ -130,6 +130,7 @@ WITH new_profile_ping AS (
      ORDER BY
        submission_timestamp
    )[SAFE_OFFSET(0)] AS attribution_dlsource,
+    ARRAY_AGG(profile_group_id ORDER BY submission_timestamp)[SAFE_OFFSET(0)] AS profile_group_id,
  FROM
    `moz-fx-data-shared-prod.telemetry.new_profile`
  WHERE
@ -270,6 +271,7 @@ shutdown_ping AS (
      ORDER BY
        submission_timestamp
    )[SAFE_OFFSET(0)] AS attribution_dlsource,
+    ARRAY_AGG(profile_group_id ORDER BY submission_timestamp)[SAFE_OFFSET(0)] AS profile_group_id,
  FROM
    `moz-fx-data-shared-prod.telemetry.first_shutdown`
  WHERE
@ -382,7 +384,8 @@ main_ping AS (
    ] AS attribution_dltoken,
    ARRAY_AGG(attribution.dlsource RESPECT NULLS ORDER BY submission_date)[
      SAFE_OFFSET(0)
-    ] AS attribution_dlsource
+    ] AS attribution_dlsource,
+    ARRAY_AGG(profile_group_id ORDER BY submission_date)[SAFE_OFFSET(0)] AS profile_group_id,
  FROM
    `moz-fx-data-shared-prod.telemetry_derived.clients_daily_v6`
  WHERE
@ -508,7 +511,47 @@ _current AS (
 ),
 _previous AS (
  SELECT
-    *
+    client_id,
+    sample_id,
+    first_seen_date,
+    second_seen_date,
+    architecture,
+    app_build_id,
+    app_name,
+    locale,
+    platform_version,
+    vendor,
+    app_version,
+    xpcom_abi,
+    document_id,
+    distribution_id,
+    partner_distribution_version,
+    partner_distributor,
+    partner_distributor_channel,
+    partner_id,
+    attribution_campaign,
+    attribution_content,
+    attribution_experiment,
+    attribution_medium,
+    attribution_source,
+    attribution_ua,
+    engine_data_load_path,
+    engine_data_name,
+    engine_data_origin,
+    engine_data_submission_url,
+    apple_model_id,
+    city,
+    db_version,
+    subdivision1,
+    normalized_channel,
+    country,
+    normalized_os,
+    normalized_os_version,
+    startup_profile_selection_reason,
+    attribution_dltoken,
+    attribution_dlsource,
+    profile_group_id,
+    metadata
  FROM
    `moz-fx-data-shared-prod.telemetry_derived.clients_first_seen_v2`
 )
--- a/tests/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/moz-fx-data-shared-prod.telemetry.first_shutdown.schema.json
+++ b/tests/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/moz-fx-data-shared-prod.telemetry.first_shutdown.schema.json
@ -268,5 +268,9 @@
    ],
    "type": "RECORD",
    "name": "payload"
+  },
+  {
+    "type": "STRING",
+    "name": "profile_group_id"
  }
 ]
--- a/tests/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/moz-fx-data-shared-prod.telemetry.new_profile.schema.json
+++ b/tests/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/moz-fx-data-shared-prod.telemetry.new_profile.schema.json
@ -268,5 +268,9 @@
    ],
    "type": "RECORD",
    "name": "payload"
+  },
+  {
+    "type": "STRING",
+    "name": "profile_group_id"
  }
 ]
--- a/tests/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/moz-fx-data-shared-prod.telemetry_derived.clients_daily_v6.schema.json
+++ b/tests/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/moz-fx-data-shared-prod.telemetry_derived.clients_daily_v6.schema.json
@ -164,5 +164,9 @@
  {
    "type": "STRING",
    "name": "startup_profile_selection_first_ping_only"
+  },
+  {
+    "type": "STRING",
+    "name": "profile_group_id"
  }
 ]
--- a/tests/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/moz-fx-data-shared-prod.telemetry_derived.clients_first_seen_v2.schema.json
+++ b/tests/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/moz-fx-data-shared-prod.telemetry_derived.clients_first_seen_v2.schema.json
@ -176,5 +176,9 @@
    ],
    "name": "metadata",
    "type": "RECORD"
+  },
+  {
+    "type": "STRING",
+    "name": "profile_group_id"
  }
 ]
--- a/tests/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/test_seen_dates/expect.yaml
+++ b/tests/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/test_seen_dates/expect.yaml
@ -14,6 +14,7 @@
  second_seen_date: 2022-07-01
  sample_id: 10
  normalized_channel: release
+  profile_group_id: profile-group-id-1
  metadata:
    first_seen_date_source_ping: new_profile
    reported_main_ping: true
@ -23,6 +24,7 @@
  first_seen_date: 2022-07-01
  sample_id: 20
  attribution_source: source2
+  profile_group_id: profile-group-id-2
  metadata:
    first_seen_date_source_ping: new_profile
    reported_main_ping: true
@ -33,6 +35,7 @@
  second_seen_date: 2022-07-01
  sample_id: 30
  normalized_channel: beta
+  profile_group_id: profile-group-id-3
  metadata:
    first_seen_date_source_ping: new_profile
    reported_new_profile_ping: true
@ -41,6 +44,7 @@
  sample_id: 40
  first_seen_date: 2022-06-29
  second_seen_date: 2022-06-30
+  profile_group_id: profile-group-id-4
  metadata:
    first_seen_date_source_ping: new_profile
    reported_new_profile_ping: true
@ -49,6 +53,7 @@
 - client_id: client-5
  sample_id: 50
  first_seen_date: 2022-06-25
+  profile_group_id: profile-group-id-5
  metadata:
    first_seen_date_source_ping: new_profile
    reported_new_profile_ping: true
--- a/tests/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/test_seen_dates/moz-fx-data-shared-prod.telemetry.first_shutdown.yaml
+++ b/tests/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/test_seen_dates/moz-fx-data-shared-prod.telemetry.first_shutdown.yaml
@ -2,6 +2,8 @@
 - client_id: client-3
  sample_id: 30
  submission_timestamp: '2022-07-01 01:00:00'
+  profile_group_id: profile-group-id-3
 - client_id: client-4
  sample_id: 40
  submission_timestamp: '2022-07-01 04:00:00'
+  profile_group_id: profile-group-id-4
--- a/tests/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/test_seen_dates/moz-fx-data-shared-prod.telemetry.new_profile.yaml
+++ b/tests/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/test_seen_dates/moz-fx-data-shared-prod.telemetry.new_profile.yaml
@ -2,6 +2,7 @@
 - client_id: client-2
  sample_id: 20
  submission_timestamp: '2022-07-01 00:02:00'
+  profile_group_id: profile-group-id-2
  environment:
    settings:
      attribution:
@ -9,12 +10,14 @@
 - client_id: client-2
  sample_id: 20
  submission_timestamp: '2022-07-01 01:02:00'
+  profile_group_id: profile-group-id-2
  environment:
    settings:
      attribution:
        campaign: campaign2
 - client_id: client-5
  sample_id: 50
+  profile_group_id: profile-group-id-5
  submission_timestamp: '2022-07-01 01:00:00'
  environment:
    build:
--- a/tests/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/test_seen_dates/moz-fx-data-shared-prod.telemetry_derived.clients_daily_v6.yaml
+++ b/tests/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/test_seen_dates/moz-fx-data-shared-prod.telemetry_derived.clients_daily_v6.yaml
@ -5,13 +5,16 @@
  locale: abc1
  submission_date: '2022-07-01'
  submission_timestamp_min: '2022-07-01 00:01:00'
+  profile_group_id: profile-group-id-1
 - client_id: client-2
  sample_id: 20
  submission_date: '2022-07-01'
  submission_timestamp_min: '2022-07-01 00:03:00'
  attribution:
    campaign: campaign2
+  profile_group_id: profile-group-id-2
 - client_id: client-3
  sample_id: 30
  submission_date: '2022-07-01'
  submission_timestamp_min: '2022-07-01 13:00:00'
+  profile_group_id: profile-group-id-3
--- a/tests/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/test_seen_dates/moz-fx-data-shared-prod.telemetry_derived.clients_first_seen_v2.yaml
+++ b/tests/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/test_seen_dates/moz-fx-data-shared-prod.telemetry_derived.clients_first_seen_v2.yaml
@ -3,6 +3,7 @@
  sample_id: 10
  first_seen_date: 2022-06-30
  normalized_channel: release
+  profile_group_id: profile-group-id-1
  metadata:
    first_seen_date_source_ping: new_profile
    reported_main_ping: false
@ -12,6 +13,7 @@
  sample_id: 30
  first_seen_date: 2022-06-30
  normalized_channel: beta
+  profile_group_id: profile-group-id-3
  metadata:
    first_seen_date_source_ping: new_profile
    reported_new_profile_ping: true
@ -20,6 +22,7 @@
  sample_id: 40
  first_seen_date: 2022-06-29
  second_seen_date: 2022-06-30
+  profile_group_id: profile-group-id-4
  metadata:
    first_seen_date_source_ping: new_profile
    reported_new_profile_ping: true
@ -28,6 +31,7 @@
 - client_id: client-5
  sample_id: 50
  first_seen_date: 2022-06-25
+  profile_group_id: profile-group-id-5
  metadata:
    first_seen_date_source_ping: new_profile
    reported_new_profile_ping: true
--- a/tests/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/test_source_pings/expect.yaml
+++ b/tests/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/test_source_pings/expect.yaml
@ -1,5 +1,6 @@
 ---
 - client_id: client-1
+  profile_group_id: profile-group-id-1
  first_seen_date: 2023-06-01
  city: city1
  sample_id: 10
@ -9,6 +10,7 @@
    reported_new_profile_ping: false
    reported_shutdown_ping: false
 - client_id: client-3
+  profile_group_id: profile-group-id-3
  first_seen_date: 2023-06-01
  attribution_dltoken: token3
  sample_id: 30
@ -18,6 +20,7 @@
    reported_new_profile_ping: true
    reported_shutdown_ping: true
 - client_id: client-4
+  profile_group_id: profile-group-id-4
  sample_id: 40
  first_seen_date: 2023-05-14
  second_seen_date: 2023-06-01
--- a/tests/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/test_source_pings/moz-fx-data-shared-prod.telemetry.first_shutdown.yaml
+++ b/tests/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/test_source_pings/moz-fx-data-shared-prod.telemetry.first_shutdown.yaml
@ -1,4 +1,5 @@
 ---
 - client_id: client-3
+  profile_group_id: profile-group-id-3
  sample_id: 30
  submission_timestamp: '2023-06-01 10:03:00'
--- a/tests/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/test_source_pings/moz-fx-data-shared-prod.telemetry.new_profile.yaml
+++ b/tests/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/test_source_pings/moz-fx-data-shared-prod.telemetry.new_profile.yaml
@ -1,6 +1,7 @@
 ---
 - client_id: client-3
  sample_id: 30
+  profile_group_id: profile-group-id-3
  submission_timestamp: '2023-06-01 12:03:00'
  environment:
    settings:
--- a/tests/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/test_source_pings/moz-fx-data-shared-prod.telemetry_derived.clients_daily_v6.yaml
+++ b/tests/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/test_source_pings/moz-fx-data-shared-prod.telemetry_derived.clients_daily_v6.yaml
@ -4,19 +4,23 @@
  submission_date: '2023-06-01'
  submission_timestamp_min: '2023-06-01 00:03:00'
  city: city1
+  profile_group_id: profile-group-id-1
 - client_id: client-2
  sample_id: 20
  submission_date: '2023-05-28'
  submission_timestamp_min: '2023-05-28 00:03:00'
  attribution:
    campaign: campaign2
+  profile_group_id: profile-group-id-2
 - client_id: client-3
  sample_id: 30
  submission_date: '2023-06-01'
  submission_timestamp_min: '2023-06-01 00:02:00'
+  profile_group_id: profile-group-id-3
 - client_id: client-4
  sample_id: 40
  submission_date: '2023-06-01'
  submission_timestamp_min: '2023-06-01 00:04:00'
  attribution:
    campaign: campaign44
+  profile_group_id: profile-group-id-4
--- a/tests/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/test_source_pings/moz-fx-data-shared-prod.telemetry_derived.clients_first_seen_v2.yaml
+++ b/tests/sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v2/test_source_pings/moz-fx-data-shared-prod.telemetry_derived.clients_first_seen_v2.yaml
@ -1,5 +1,6 @@
 ---
 - client_id: client-4
+  profile_group_id: profile-group-id-4
  sample_id: 40
  first_seen_date: 2023-05-14
  normalized_channel: nightly