adding checks to all models (#5500)

* adding checks to all models

* updating sql to remove udf

* formatting sql

* updating checks and adding suppressions back in for now

* updating checks

* updating metadata to add referenced tables

* updating checks to remove circular reference and adding changed subscriptions

* reformatting

* fixing format issues

sql/moz-fx-data-shared-prod/braze_derived/newsletters_v1/checks.sql

@@ -0,0 +1,25 @@
+-- raw SQL checks
+-- checking to see if there is new data since the last run
+-- if not, fail or we will have blank sync tables
+
+#fail
+ASSERT(
+  SELECT
+    COUNT(1)
+  FROM
+    `moz-fx-data-shared-prod.braze_derived.newsletters_v1`,
+    UNNEST(newsletters) AS newsletters
+  WHERE
+    newsletters.update_timestamp > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 15 HOUR)
+) > 0;
+
+-- macro checks
+
+#fail
+{{ not_null(["external_id"]) }} -- to do: add array values
+
+#fail
+{{ is_unique(["external_id"]) }}
+
+#fail
+{{ min_row_count(1) }}

sql/moz-fx-data-shared-prod/braze_derived/products_v1/checks.sql

@@ -0,0 +1,27 @@
+-- raw SQL checks
+-- checking to see if there is new data since the last run
+-- if not, fail or we will have blank sync tables
+
+#fail
+ASSERT(
+-- Retrieves the maximum subscription updated timestamp from the last run to only
+-- select recently changed records
+  SELECT
+    COUNT(1)
+  FROM
+    `moz-fx-data-shared-prod.braze_derived.products_v1`,
+    UNNEST(products) AS products
+  WHERE
+    products.subscription_updated_at > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 15 HOUR)
+) > 0;
+
+-- macro checks
+
+#fail
+{{ not_null(["external_id"]) }} -- to do: add array values
+
+#fail
+{{ is_unique(["external_id"]) }}
+
+#fail
+{{ min_row_count(1) }}

sql/moz-fx-data-shared-prod/braze_derived/subscriptions_map_v1/checks.sql

@@ -0,0 +1,18 @@
+-- raw sql checks
+ASSERT(SELECT COUNT(*) FROM `moz-fx-data-shared-prod.braze_derived.users_v1`) = (
+  SELECT
+    COUNT(*)
+  FROM
+    `moz-fx-data-shared-prod.braze_derived.user_profiles_v1`
+);
+
+-- macro checks
+
+#fail
+{{ not_null(["braze_subscription_name", "description", "mozilla_subscription_id", "firefox_subscription_id", "mozilla_dev_subscription_id", "basket_slug"]) }}
+
+#fail
+{{ min_row_count(1) }}
+
+#fail
+{{ is_unique(["mozilla_subscription_id", "firefox_subscription_id", "mozilla_dev_subscription_id"]) }}

sql/moz-fx-data-shared-prod/braze_derived/subscriptions_v1/checks.sql

@@ -0,0 +1,10 @@
+-- macro checks
+
+#fail
+{{ not_null(["external_id"]) }} -- to do: add array values
+
+#fail
+{{ is_unique(["external_id"]) }}
+
+#fail
+{{ min_row_count(1) }}

sql/moz-fx-data-shared-prod/braze_derived/user_profiles_v1/checks.sql

@@ -0,0 +1,10 @@
+-- macro checks
+
+#fail
+{{ not_null(["external_id", "email", "email_subscribe", "has_fxa", "create_timestamp", "update_timestamp"]) }}
+
+#fail
+{{ min_row_count(75000000) }}
+
+#fail
+{{ is_unique(["external_id", "email", "fxa_id_sha256"]) }}

sql/moz-fx-data-shared-prod/braze_derived/users_v1/checks.sql

@@ -0,0 +1,24 @@
+-- raw SQL checks
+-- checking to see if there is new data since the last run
+-- if not, fail or we will have blank sync tables
+
+#fail
+ASSERT(
+  SELECT
+    COUNT(1)
+  FROM
+    `moz-fx-data-shared-prod.braze_derived.users_v1` AS users
+  WHERE
+    users.update_timestamp > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 15 HOUR)
+) > 0;
+
+-- macro checks
+
+#fail
+{{ not_null(["external_id", "email", "email_subscribe", "has_fxa", "create_timestamp", "update_timestamp"]) }}
+
+#fail
+{{ min_row_count(85000000) }}
+
+#fail
+{{ is_unique(["external_id", "email", "fxa_id_sha256"]) }}

sql/moz-fx-data-shared-prod/braze_derived/waitlists_v1/checks.sql

@@ -0,0 +1,23 @@
+-- raw SQL checks
+-- checking to see if there is new data since the last run
+-- if not, fail or we will have blank sync tables
+
+#fail
+ASSERT(
+  SELECT
+    COUNT(1)
+  FROM
+    `moz-fx-data-shared-prod.braze_derived.waitlists_v1`,
+    UNNEST(waitlists) AS waitlists
+  WHERE
+    waitlists.update_timestamp > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 15 HOUR)
+) > 0;
+
+#fail
+{{ not_null(["external_id"]) }} -- to do: add array values
+
+#fail
+{{ is_unique(["external_id"]) }}
+
+#fail
+{{ min_row_count(1) }}

sql/moz-fx-data-shared-prod/braze_external/changed_firefox_subscriptions_sync_v1/checks.sql

@@ -0,0 +1,10 @@
+-- macro checks
+
+#warn
+{{ not_null(["EXTERNAL_ID", "UPDATED_AT", "PAYLOAD"]) }}
+
+#warn
+{{ min_row_count(1) }}
+
+#warn
+{{ is_unique(["EXTERNAL_ID"]) }}

sql/moz-fx-data-shared-prod/braze_external/changed_newsletters_sync_v1/checks.sql

@@ -0,0 +1,10 @@
+-- macro checks
+
+#warn
+{{ not_null(["EXTERNAL_ID", "UPDATED_AT", "PAYLOAD"]) }}
+
+#warn
+{{ min_row_count(1) }}
+
+#warn
+{{ is_unique(["EXTERNAL_ID"]) }}

sql/moz-fx-data-shared-prod/braze_external/changed_products_sync_v1/checks.sql

@@ -0,0 +1,10 @@
+-- macro checks
+
+#warn
+{{ not_null(["EXTERNAL_ID", "UPDATED_AT", "PAYLOAD"]) }}
+
+#warn
+{{ min_row_count(1) }}
+
+#warn
+{{ is_unique(["EXTERNAL_ID"]) }}

sql/moz-fx-data-shared-prod/braze_external/changed_subscriptions_v1/checks.sql

@@ -0,0 +1,25 @@
+-- raw SQL checks
+-- checking to see if there is new data since the last run
+-- if not, fail or we will have blank sync tables
+
+#fail
+ASSERT(
+  SELECT
+    COUNT(1)
+  FROM
+    `moz-fx-data-shared-prod.braze_derived.subscriptions_v1`,
+    UNNEST(subscriptions) AS subscriptions
+  WHERE
+    subscriptions.update_timestamp > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 15 HOUR)
+) > 0;
+
+-- macro checks
+
+#warn
+{{ not_null(["external_id"]) }}
+
+#warn
+{{ min_row_count(1) }}
+
+#warn
+{{ is_unique(["external_id"]) }}

sql/moz-fx-data-shared-prod/braze_external/changed_subscriptions_v1/metadata.yaml

@@ -0,0 +1,19 @@
+friendly_name: Braze Changed Subscriptions Sync | Firefox Workspace
+description: |-
+  Changes in subscriptions in Braze from the previous day - mapped to the Firefox workspace.
+  In order to consume the least amount of data points, we only want to sync rows changed
+  since the last sync. This table holds all of the changed subscription data that syncs to braze.
+
+
+  See https://mozilla-hub.atlassian.net/browse/DENG-3182
+owners:
+- cbeck@mozilla.com
+labels:
+  incremental: false
+  schedule: daily
+  owner: cbeck
+bigquery:
+  time_partitioning: null
+scheduling:
+  dag_name: bqetl_braze
+  date_partition_parameter: null

sql/moz-fx-data-shared-prod/braze_external/changed_subscriptions_v1/query.sql

@@ -0,0 +1,36 @@
+WITH max_update_timestamp AS (
+  SELECT
+    MAX(subscriptions.update_timestamp) AS latest_subscription_updated_at
+  FROM
+    `moz-fx-data-shared-prod.braze_external.changed_subscriptions_v1` AS changed,
+    UNNEST(changed.subscriptions) AS subscriptions
+),
+filtered_subscriptions AS (
+  SELECT
+    subscriptions.external_id,
+    ARRAY(
+      SELECT AS STRUCT
+        subscriptions_array.subscription_name AS subscription_name,
+        subscriptions_array.firefox_subscription_id AS firefox_subscription_id,
+        subscriptions_array.mozilla_subscription_id AS mozilla_subscription_id,
+        subscriptions_array.mozilla_dev_subscription_id AS mozilla_dev_subscription_id,
+        subscriptions_array.subscription_state AS subscription_state,
+        subscriptions_array.update_timestamp AS update_timestamp
+      FROM
+        UNNEST(subscriptions.subscriptions) AS subscriptions_array
+      WHERE
+        subscriptions_array.update_timestamp > max_update_timestamp.latest_subscription_updated_at
+        AND subscriptions_array.update_timestamp IS NOT NULL
+    ) AS subscriptions
+  FROM
+    `moz-fx-data-shared-prod.braze_derived.subscriptions_v1` AS subscriptions
+  CROSS JOIN
+    max_update_timestamp
+)
+SELECT
+  external_id,
+  subscriptions
+FROM
+  filtered_subscriptions
+WHERE
+  ARRAY_LENGTH(subscriptions) > 0;

sql/moz-fx-data-shared-prod/braze_external/changed_subscriptions_v1/schema.yaml

@@ -0,0 +1,26 @@
+fields:
+- mode: NULLABLE
+  name: external_id
+  type: STRING
+- fields:
+  - mode: NULLABLE
+    name: subscription_name
+    type: STRING
+  - mode: NULLABLE
+    name: firefox_subscription_id
+    type: STRING
+  - mode: NULLABLE
+    name: mozilla_subscription_id
+    type: STRING
+  - mode: NULLABLE
+    name: mozilla_dev_subscription_id
+    type: STRING
+  - mode: NULLABLE
+    name: subscription_state
+    type: STRING
+  - mode: NULLABLE
+    name: update_timestamp
+    type: TIMESTAMP
+  mode: REPEATED
+  name: subscriptions
+  type: RECORD

sql/moz-fx-data-shared-prod/braze_external/changed_users_sync_v1/checks.sql

@@ -0,0 +1,10 @@
+-- macro checks
+
+#warn
+{{ not_null(["EXTERNAL_ID", "UPDATED_AT", "PAYLOAD"]) }}
+
+#warn
+{{ min_row_count(1) }}
+
+#warn
+{{ is_unique(["EXTERNAL_ID"]) }}

sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/checks.sql

@@ -0,0 +1,10 @@
+-- macro checks
+
+#warn
+{{ not_null(["external_id", "status", "email", "create_timestamp", "update_timestamp"]) }}
+
+#warn
+{{ min_row_count(1) }}
+
+#warn
+{{ is_unique(["external_id", "email", "fxa_id_sha256"]) }}

sql/moz-fx-data-shared-prod/braze_external/changed_waitlists_sync_v1/checks.sql

@@ -0,0 +1,10 @@
+-- macro checks
+
+#warn
+{{ not_null(["EXTERNAL_ID", "UPDATED_AT", "PAYLOAD"]) }}
+
+#warn
+{{ min_row_count(1) }}
+
+#warn
+{{ is_unique(["EXTERNAL_ID"]) }}

sql/moz-fx-data-shared-prod/braze_external/delete_users_sync_v1/checks.sql

@@ -0,0 +1,10 @@
+-- macro checks
+
+#warn
+{{ not_null(["EXTERNAL_ID"]) }}
+
+#warn
+{{ min_row_count(1) }}
+
+#warn
+{{ is_unique(["EXTERNAL_ID"]) }}

sql/moz-fx-data-shared-prod/marketing_suppression_list_derived/main_suppression_list_v1/checks.sql

@@ -0,0 +1,8 @@
+#fail
+{{ not_null(["email", "suppressed_timestamp"]) }}
+
+#fail
+{{ min_row_count(1) }}
+
+#warn
+{{ is_unique(["email"]) }}