Implement simple generic active subscriptions table for KPIs (#4149)

bigquery_etl/format_sql/tokenizer.py

@@ -783,7 +783,7 @@ class JinjaBlockMiddle(JinjaBlockEnd, JinjaBlockStart):
     pattern = re.compile(r"{% *(elif|else)\b.*?%}", re.DOTALL)
 
 
-class JinjaComment(Token):
+class JinjaComment(Comment):
     """Jinja comment delimiters {# #}.
 
     May be followed by no whitespace or a new line and increased indent.

dags/bqetl_subplat.py

@@ -939,6 +939,18 @@ with DAG(
         task_concurrency=1,
     )
 
+    subscription_platform_derived__active_subscriptions__v1 = bigquery_etl_query(
+        task_id="subscription_platform_derived__active_subscriptions__v1",
+        destination_table="active_subscriptions_v1",
+        dataset_id="subscription_platform_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="srose@mozilla.com",
+        email=["srose@mozilla.com", "telemetry-alerts@mozilla.com"],
+        date_partition_parameter=None,
+        depends_on_past=False,
+        task_concurrency=1,
+    )
+
     subscription_platform_derived__apple_subscriptions__v1 = bigquery_etl_query(
         task_id="subscription_platform_derived__apple_subscriptions__v1",
         destination_table="apple_subscriptions_v1",
@@ -1379,6 +1391,42 @@ with DAG(
 
     stripe_external__tax_rate__v1.set_upstream(fivetran_stripe_sync_wait)
 
+    subscription_platform_derived__active_subscriptions__v1.set_upstream(
+        hubs_derived__active_subscription_ids__v1
+    )
+
+    subscription_platform_derived__active_subscriptions__v1.set_upstream(
+        hubs_derived__active_subscriptions__v1
+    )
+
+    subscription_platform_derived__active_subscriptions__v1.set_upstream(
+        hubs_derived__subscriptions__v1
+    )
+
+    subscription_platform_derived__active_subscriptions__v1.set_upstream(
+        mozilla_vpn_derived__active_subscription_ids__v1
+    )
+
+    subscription_platform_derived__active_subscriptions__v1.set_upstream(
+        mozilla_vpn_derived__active_subscriptions__v1
+    )
+
+    subscription_platform_derived__active_subscriptions__v1.set_upstream(
+        mozilla_vpn_derived__all_subscriptions__v1
+    )
+
+    subscription_platform_derived__active_subscriptions__v1.set_upstream(
+        relay_derived__active_subscription_ids__v1
+    )
+
+    subscription_platform_derived__active_subscriptions__v1.set_upstream(
+        relay_derived__active_subscriptions__v1
+    )
+
+    subscription_platform_derived__active_subscriptions__v1.set_upstream(
+        relay_derived__subscriptions__v1
+    )
+
     subscription_platform_derived__apple_subscriptions__v1.set_upstream(
         mozilla_vpn_derived__guardian_apple_events__v1
     )

sql/moz-fx-data-shared-prod/subscription_platform/active_subscriptions/view.sql

@@ -0,0 +1,7 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.subscription_platform.active_subscriptions`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod`.subscription_platform_derived.active_subscriptions_v1

sql/moz-fx-data-shared-prod/subscription_platform_derived/active_subscriptions_v1/metadata.yaml

@@ -0,0 +1,18 @@
+friendly_name: Active Stripe Subscriptions
+description: |
+  Active Stripe subscriptions for products included in KPIs.
+owners:
+  - srose@mozilla.com
+labels:
+  schedule: daily
+  incremental: false
+scheduling:
+  dag_name: bqetl_subplat
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+bigquery:
+  time_partitioning:
+    type: day
+    field: active_date
+    require_partition_filter: false

sql/moz-fx-data-shared-prod/subscription_platform_derived/active_subscriptions_v1/query.sql

@@ -0,0 +1,52 @@
+{#- format off #}
+WITH product_union AS (
+{%- for product in ["mozilla_vpn", "relay", "hubs"] %}
+  SELECT
+    active_date,
+    plan_id,
+    country,
+    country_name,
+    provider,
+    plan_amount,
+    plan_currency,
+    plan_interval,
+    plan_interval_count,
+    product_id,
+    product_name,
+    pricing_plan,
+    TO_JSON_STRING(promotion_codes) AS promotion_codes,
+    promotion_discounts_amount,
+    `count`
+  FROM
+    `moz-fx-data-shared-prod`.{{product}}.active_subscriptions
+{%- if product == "mozilla_vpn" %}
+  WHERE
+    -- in order to avoid double counting, only include bundles via relay
+    product_name NOT LIKE "%Relay%"
+{%- endif %}
+{%- if not loop.last %}
+  UNION ALL
+{%- endif %}
+{%- endfor %}
+{#- format on #}
+)
+SELECT
+  * EXCEPT (`count`) REPLACE(JSON_VALUE_ARRAY(promotion_codes) AS promotion_codes),
+  SUM(`count`) AS `count`,
+FROM
+  product_union
+GROUP BY
+  active_date,
+  plan_id,
+  country,
+  country_name,
+  provider,
+  plan_amount,
+  plan_currency,
+  plan_interval,
+  plan_interval_count,
+  product_id,
+  product_name,
+  pricing_plan,
+  product_union.promotion_codes,
+  promotion_discounts_amount

sql/moz-fx-data-shared-prod/subscription_platform_derived/active_subscriptions_v1/schema.yaml

@@ -0,0 +1,46 @@
+fields:
+- name: active_date
+  type: DATE
+  mode: NULLABLE
+- name: plan_id
+  type: STRING
+  mode: NULLABLE
+- name: country
+  type: STRING
+  mode: NULLABLE
+- name: country_name
+  type: STRING
+  mode: NULLABLE
+- name: provider
+  type: STRING
+  mode: NULLABLE
+- name: plan_amount
+  type: INTEGER
+  mode: NULLABLE
+- name: plan_currency
+  type: STRING
+  mode: NULLABLE
+- name: plan_interval
+  type: STRING
+  mode: NULLABLE
+- name: plan_interval_count
+  type: INTEGER
+  mode: NULLABLE
+- name: product_id
+  type: STRING
+  mode: NULLABLE
+- name: product_name
+  type: STRING
+  mode: NULLABLE
+- name: pricing_plan
+  type: STRING
+  mode: NULLABLE
+- name: promotion_codes
+  type: STRING
+  mode: REPEATED
+- name: promotion_discounts_amount
+  type: INTEGER
+  mode: NULLABLE
+- name: count
+  type: INTEGER
+  mode: NULLABLE