Add Hubs subscriptions ETL (DENG-681) (#3632)

2023-03-03 15:54:17 -08:00 · 2023-03-03 15:54:17 -08:00 · 88730fa005
--- a/bigquery_etl/dryrun.py
+++ b/bigquery_etl/dryrun.py
@ -133,6 +133,7 @@ SKIP = {
    "sql/moz-fx-data-shared-prod/mozilla_vpn_derived/protected_v1/init.sql",
    "sql/moz-fx-data-shared-prod/mozilla_vpn_derived/add_device_events_v1/init.sql",
    "sql/moz-fx-data-shared-prod/mozilla_vpn_external/devices_v1/init.sql",
+    "sql/moz-fx-data-shared-prod/hubs_derived/subscriptions_v1/query.sql",
    "sql/moz-fx-data-shared-prod/relay_derived/subscriptions_v1/query.sql",
    "sql/moz-fx-data-shared-prod/fenix_derived/google_ads_campaign_cost_breakdowns_v1/query.sql",
    *glob.glob("sql/moz-fx-data-shared-prod/search_terms*/**/*.sql", recursive=True),
--- a/dags/bqetl_subplat.py
+++ b/dags/bqetl_subplat.py
@ -100,6 +100,54 @@ with DAG(
        task_concurrency=1,
    )

+    hubs_derived__active_subscription_ids__v1 = bigquery_etl_query(
+        task_id="hubs_derived__active_subscription_ids__v1",
+        destination_table='active_subscription_ids_v1${{ macros.ds_format(macros.ds_add(ds, -7), "%Y-%m-%d", "%Y%m%d") }}',
+        dataset_id="hubs_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="srose@mozilla.com",
+        email=["srose@mozilla.com", "telemetry-alerts@mozilla.com"],
+        date_partition_parameter=None,
+        depends_on_past=True,
+        parameters=["date:DATE:{{macros.ds_add(ds, -7)}}"],
+    )
+
+    hubs_derived__active_subscriptions__v1 = bigquery_etl_query(
+        task_id="hubs_derived__active_subscriptions__v1",
+        destination_table='active_subscriptions_v1${{ macros.ds_format(macros.ds_add(ds, -7), "%Y-%m-%d", "%Y%m%d") }}',
+        dataset_id="hubs_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="srose@mozilla.com",
+        email=["srose@mozilla.com", "telemetry-alerts@mozilla.com"],
+        date_partition_parameter=None,
+        depends_on_past=False,
+        parameters=["date:DATE:{{macros.ds_add(ds, -7)}}"],
+    )
+
+    hubs_derived__subscription_events__v1 = bigquery_etl_query(
+        task_id="hubs_derived__subscription_events__v1",
+        destination_table='subscription_events_v1${{ macros.ds_format(macros.ds_add(ds, -8), "%Y-%m-%d", "%Y%m%d") }}',
+        dataset_id="hubs_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="srose@mozilla.com",
+        email=["srose@mozilla.com", "telemetry-alerts@mozilla.com"],
+        date_partition_parameter=None,
+        depends_on_past=False,
+        parameters=["date:DATE:{{macros.ds_add(ds, -8)}}"],
+    )
+
+    hubs_derived__subscriptions__v1 = bigquery_etl_query(
+        task_id="hubs_derived__subscriptions__v1",
+        destination_table="subscriptions_v1",
+        dataset_id="hubs_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="srose@mozilla.com",
+        email=["srose@mozilla.com", "telemetry-alerts@mozilla.com"],
+        date_partition_parameter=None,
+        depends_on_past=False,
+        task_concurrency=1,
+    )
+
    mozilla_vpn_derived__active_subscription_ids__v1 = bigquery_etl_query(
        task_id="mozilla_vpn_derived__active_subscription_ids__v1",
        destination_table='active_subscription_ids_v1${{ macros.ds_format(macros.ds_add(ds, -7), "%Y-%m-%d", "%Y%m%d") }}',
@ -703,6 +751,24 @@ with DAG(
    cjms_bigquery__subscriptions__v1.set_upstream(cjms_bigquery__flows__v1)
    cjms_bigquery__subscriptions__v1.set_upstream(fivetran_stripe_sync_wait)

+    hubs_derived__active_subscription_ids__v1.set_upstream(
+        hubs_derived__subscriptions__v1
+    )
+
+    hubs_derived__active_subscriptions__v1.set_upstream(
+        hubs_derived__active_subscription_ids__v1
+    )
+
+    hubs_derived__active_subscriptions__v1.set_upstream(hubs_derived__subscriptions__v1)
+
+    hubs_derived__subscription_events__v1.set_upstream(
+        hubs_derived__active_subscription_ids__v1
+    )
+
+    hubs_derived__subscription_events__v1.set_upstream(hubs_derived__subscriptions__v1)
+
+    hubs_derived__subscriptions__v1.set_upstream(fivetran_stripe_sync_wait)
+
    mozilla_vpn_derived__active_subscription_ids__v1.set_upstream(
        mozilla_vpn_derived__all_subscriptions__v1
    )
--- a/sql/moz-fx-data-shared-prod/hubs/active_subscription_ids/view.sql
+++ b/sql/moz-fx-data-shared-prod/hubs/active_subscription_ids/view.sql
@ -0,0 +1,23 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.hubs.active_subscription_ids`
+AS
+WITH max_active_date AS (
+  SELECT AS VALUE
+    MAX(active_date)
+  FROM
+    `moz-fx-data-shared-prod`.hubs_derived.active_subscription_ids_v1
+)
+SELECT
+  active_subscription_ids_live.*
+FROM
+  `moz-fx-data-shared-prod`.hubs_derived.active_subscription_ids_live
+CROSS JOIN
+  max_active_date
+WHERE
+  -- static partition filter not needed because live view doesn't use date partitioned tables
+  active_date > max_active_date
+UNION ALL
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod`.hubs_derived.active_subscription_ids_v1
--- a/sql/moz-fx-data-shared-prod/hubs/active_subscriptions/view.sql
+++ b/sql/moz-fx-data-shared-prod/hubs/active_subscriptions/view.sql
@ -0,0 +1,22 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.hubs.active_subscriptions`
+AS
+WITH max_agg_date AS (
+  SELECT AS VALUE
+    MAX(active_date)
+  FROM
+    `moz-fx-data-shared-prod`.hubs_derived.active_subscriptions_v1
+)
+SELECT
+  active_subscriptions_live.*
+FROM
+  `moz-fx-data-shared-prod`.hubs_derived.active_subscriptions_live
+CROSS JOIN
+  max_agg_date
+WHERE
+  active_date > max_agg_date
+UNION ALL
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod`.hubs_derived.active_subscriptions_v1
--- a/sql/moz-fx-data-shared-prod/hubs/dataset_metadata.yaml
+++ b/sql/moz-fx-data-shared-prod/hubs/dataset_metadata.yaml
@ -0,0 +1,10 @@
+friendly_name: Hubs
+description: |-
+  Data related to the Hubs service
+dataset_base_acl: view
+user_facing: true
+labels: {}
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential
--- a/sql/moz-fx-data-shared-prod/hubs/subscription_events/view.sql
+++ b/sql/moz-fx-data-shared-prod/hubs/subscription_events/view.sql
@ -0,0 +1,22 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.hubs.subscription_events`
+AS
+WITH max_agg_date AS (
+  SELECT AS VALUE
+    MAX(event_date)
+  FROM
+    `moz-fx-data-shared-prod`.hubs_derived.subscription_events_v1
+)
+SELECT
+  subscription_events_live.*
+FROM
+  `moz-fx-data-shared-prod`.hubs_derived.subscription_events_live
+CROSS JOIN
+  max_agg_date
+WHERE
+  event_date > max_agg_date
+UNION ALL
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod`.hubs_derived.subscription_events_v1
--- a/sql/moz-fx-data-shared-prod/hubs/subscriptions/view.sql
+++ b/sql/moz-fx-data-shared-prod/hubs/subscriptions/view.sql
@ -0,0 +1,7 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.hubs.subscriptions`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod`.hubs_derived.subscriptions_v1
--- a/sql/moz-fx-data-shared-prod/hubs_derived/active_subscription_ids_live/view.sql
+++ b/sql/moz-fx-data-shared-prod/hubs_derived/active_subscription_ids_live/view.sql
@ -0,0 +1,23 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.hubs_derived.active_subscription_ids_live`
+AS
+SELECT
+  active_date,
+  subscription_id,
+FROM
+  `moz-fx-data-shared-prod`.hubs.subscriptions
+CROSS JOIN
+  UNNEST(
+    GENERATE_DATE_ARRAY(
+      DATE(subscription_start_date),
+      GREATEST(DATE(subscription_start_date), DATE(end_date) - 1)
+    )
+  ) AS active_date
+WHERE
+  subscription_start_date IS NOT NULL
+  AND DATE(subscription_start_date) < (
+    SELECT
+      DATE(MAX(end_date))
+    FROM
+      `moz-fx-data-shared-prod`.hubs.subscriptions
+  )
--- a/sql/moz-fx-data-shared-prod/hubs_derived/active_subscription_ids_v1/metadata.yaml
+++ b/sql/moz-fx-data-shared-prod/hubs_derived/active_subscription_ids_v1/metadata.yaml
@ -0,0 +1,21 @@
+friendly_name: Active Hubs Subscription IDs
+description: >
+  IDs of active Hubs subscriptions by date.
+owners:
+  - srose@mozilla.com
+labels:
+  application: hubs
+  schedule: daily
+scheduling:
+  dag_name: bqetl_subplat
+  # While this ETL doesn't depend on its own previous runs, other ETLs are built
+  # on the assumption that this is built sequentially day-by-day with no gaps.
+  depends_on_past: true
+  # delay aggregates by 7 days, to ensure data is complete
+  date_partition_offset: -7
+  date_partition_parameter: date
+bigquery:
+  time_partitioning:
+    type: day
+    field: active_date
+    require_partition_filter: false
--- a/sql/moz-fx-data-shared-prod/hubs_derived/active_subscription_ids_v1/query.sql
+++ b/sql/moz-fx-data-shared-prod/hubs_derived/active_subscription_ids_v1/query.sql
@ -0,0 +1,6 @@
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod`.hubs_derived.active_subscription_ids_live
+WHERE
+  IF(CAST(@date AS DATE) IS NULL, active_date < CURRENT_DATE - 7, active_date = @date)
--- a/sql/moz-fx-data-shared-prod/hubs_derived/active_subscription_ids_v1/schema.yaml
+++ b/sql/moz-fx-data-shared-prod/hubs_derived/active_subscription_ids_v1/schema.yaml
@ -0,0 +1,7 @@
+fields:
+- name: active_date
+  type: DATE
+  mode: NULLABLE
+- name: subscription_id
+  type: STRING
+  mode: NULLABLE
--- a/sql/moz-fx-data-shared-prod/hubs_derived/active_subscriptions_live/view.sql
+++ b/sql/moz-fx-data-shared-prod/hubs_derived/active_subscriptions_live/view.sql
@ -0,0 +1,49 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.hubs_derived.active_subscriptions_live`
+AS
+WITH subscriptions AS (
+  SELECT
+    *,
+    TO_JSON_STRING(promotion_codes) AS json_promotion_codes
+  FROM
+    `moz-fx-data-shared-prod`.hubs.subscriptions
+)
+SELECT
+  active_subscription_ids.active_date,
+  subscriptions.plan_id,
+  subscriptions.country,
+  subscriptions.country_name,
+  subscriptions.provider,
+  subscriptions.plan_amount,
+  subscriptions.billing_scheme,
+  subscriptions.plan_currency,
+  subscriptions.plan_interval,
+  subscriptions.plan_interval_count,
+  subscriptions.product_id,
+  subscriptions.product_name,
+  subscriptions.pricing_plan,
+  JSON_VALUE_ARRAY(subscriptions.json_promotion_codes) AS promotion_codes,
+  subscriptions.promotion_discounts_amount,
+  COUNT(*) AS `count`,
+FROM
+  subscriptions
+JOIN
+  `moz-fx-data-shared-prod`.hubs.active_subscription_ids
+USING
+  (subscription_id)
+GROUP BY
+  active_date,
+  plan_id,
+  country,
+  country_name,
+  provider,
+  plan_amount,
+  billing_scheme,
+  plan_currency,
+  plan_interval,
+  plan_interval_count,
+  product_id,
+  product_name,
+  pricing_plan,
+  json_promotion_codes,
+  promotion_discounts_amount
--- a/sql/moz-fx-data-shared-prod/hubs_derived/active_subscriptions_v1/metadata.yaml
+++ b/sql/moz-fx-data-shared-prod/hubs_derived/active_subscriptions_v1/metadata.yaml
@ -0,0 +1,18 @@
+friendly_name: Active Hubs Subscriptions
+description: >
+  Aggregated count of active Hubs subscriptions.
+owners:
+  - srose@mozilla.com
+labels:
+  application: hubs
+  schedule: daily
+scheduling:
+  dag_name: bqetl_subplat
+  # delay aggregates by 7 days, to ensure data is complete
+  date_partition_offset: -7
+  date_partition_parameter: date
+bigquery:
+  time_partitioning:
+    type: day
+    field: active_date
+    require_partition_filter: false
--- a/sql/moz-fx-data-shared-prod/hubs_derived/active_subscriptions_v1/query.sql
+++ b/sql/moz-fx-data-shared-prod/hubs_derived/active_subscriptions_v1/query.sql
@ -0,0 +1,6 @@
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod`.hubs_derived.active_subscriptions_live
+WHERE
+  IF(CAST(@date AS DATE) IS NULL, active_date < CURRENT_DATE - 7, active_date = @date)
--- a/sql/moz-fx-data-shared-prod/hubs_derived/active_subscriptions_v1/schema.yaml
+++ b/sql/moz-fx-data-shared-prod/hubs_derived/active_subscriptions_v1/schema.yaml
@ -0,0 +1,49 @@
+fields:
+- name: active_date
+  type: DATE
+  mode: NULLABLE
+- name: plan_id
+  type: STRING
+  mode: NULLABLE
+- name: country
+  type: STRING
+  mode: NULLABLE
+- name: country_name
+  type: STRING
+  mode: NULLABLE
+- name: provider
+  type: STRING
+  mode: NULLABLE
+- name: plan_amount
+  type: INTEGER
+  mode: NULLABLE
+- name: billing_scheme
+  type: STRING
+  mode: NULLABLE
+- name: plan_currency
+  type: STRING
+  mode: NULLABLE
+- name: plan_interval
+  type: STRING
+  mode: NULLABLE
+- name: plan_interval_count
+  type: INTEGER
+  mode: NULLABLE
+- name: product_id
+  type: STRING
+  mode: NULLABLE
+- name: product_name
+  type: STRING
+  mode: NULLABLE
+- name: pricing_plan
+  type: STRING
+  mode: NULLABLE
+- name: promotion_codes
+  type: STRING
+  mode: REPEATED
+- name: promotion_discounts_amount
+  type: INTEGER
+  mode: NULLABLE
+- name: count
+  type: INTEGER
+  mode: NULLABLE
--- a/sql/moz-fx-data-shared-prod/hubs_derived/dataset_metadata.yaml
+++ b/sql/moz-fx-data-shared-prod/hubs_derived/dataset_metadata.yaml
@ -0,0 +1,10 @@
+friendly_name: Hubs Derived
+description: |-
+  Derived data related to the Hubs service
+dataset_base_acl: derived
+user_facing: false
+labels: {}
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential
--- a/sql/moz-fx-data-shared-prod/hubs_derived/subscription_events_live/view.sql
+++ b/sql/moz-fx-data-shared-prod/hubs_derived/subscription_events_live/view.sql
@ -0,0 +1,147 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.hubs_derived.subscription_events_live`
+AS
+WITH subscriptions AS (
+  SELECT
+    *,
+    TO_JSON_STRING(promotion_codes) AS json_promotion_codes
+  FROM
+    `moz-fx-data-shared-prod`.hubs.subscriptions
+),
+max_active_date AS (
+  SELECT AS VALUE
+    MAX(active_date)
+  FROM
+    `moz-fx-data-shared-prod`.hubs.active_subscription_ids
+),
+trials AS (
+  SELECT
+    *
+  FROM
+    subscriptions
+  WHERE
+    trial_start IS NOT NULL
+    AND DATE(trial_start) <= (SELECT max_active_date FROM max_active_date)
+),
+new_trial_events AS (
+  SELECT
+    DATE(trial_start) AS event_date,
+    subscription_id,
+    "New Trial" AS event_type,
+  FROM
+    trials
+),
+cancelled_trial_events AS (
+  SELECT
+    DATE(ended_at) AS event_date,
+    subscription_id,
+    "Cancelled Trial" AS event_type,
+  FROM
+    trials
+  WHERE
+    subscription_start_date IS NULL
+    AND ended_at IS NOT NULL
+),
+new_events AS (
+  SELECT
+    active_date AS event_date,
+    subscription_id,
+    "New" AS event_type,
+  FROM
+    `moz-fx-data-shared-prod`.hubs.active_subscription_ids
+  WHERE
+    TRUE -- zetasql requires QUALIFY to be used in conjunction with WHERE, GROUP BY, or HAVING
+  QUALIFY
+    LAG(active_date) OVER (PARTITION BY subscription_id ORDER BY active_date) IS DISTINCT FROM (
+      active_date - 1
+    )
+),
+cancelled_events AS (
+  SELECT
+    active_date + 1 AS event_date,
+    subscription_id,
+    "Cancelled" AS event_type,
+  FROM
+    `moz-fx-data-shared-prod`.hubs.active_subscription_ids
+  CROSS JOIN
+    max_active_date
+  WHERE
+    TRUE -- zetasql requires QUALIFY to be used in conjunction with WHERE, GROUP BY, or HAVING
+  QUALIFY
+    LEAD(active_date) OVER (PARTITION BY subscription_id ORDER BY active_date) IS DISTINCT FROM (
+      active_date + 1
+    )
+    AND active_date < max_active_date
+),
+events AS (
+  SELECT
+    *
+  FROM
+    new_trial_events
+  UNION ALL
+  SELECT
+    *
+  FROM
+    cancelled_trial_events
+  UNION ALL
+  SELECT
+    *
+  FROM
+    new_events
+  UNION ALL
+  SELECT
+    *
+  FROM
+    cancelled_events
+)
+SELECT
+  events.event_date,
+  events.event_type,
+  CASE
+    WHEN events.event_type IN ("New Trial", "Cancelled Trial")
+      THEN events.event_type
+    WHEN events.event_type = "New"
+      THEN subscriptions.subscription_start_reason
+    WHEN events.event_type = "Cancelled"
+      THEN COALESCE(
+          subscriptions.ended_reason,
+          IF(subscriptions.provider = "Apple Store", "Cancelled by IAP", "Payment Failed")
+        )
+  END AS granular_event_type,
+  subscriptions.plan_id,
+  subscriptions.country,
+  subscriptions.country_name,
+  subscriptions.provider,
+  subscriptions.plan_amount,
+  subscriptions.billing_scheme,
+  subscriptions.plan_currency,
+  subscriptions.plan_interval,
+  subscriptions.plan_interval_count,
+  subscriptions.product_id,
+  subscriptions.product_name,
+  subscriptions.pricing_plan,
+  JSON_VALUE_ARRAY(subscriptions.json_promotion_codes) AS promotion_codes,
+  COUNT(*) AS `count`,
+FROM
+  subscriptions
+JOIN
+  events
+USING
+  (subscription_id)
+GROUP BY
+  event_date,
+  event_type,
+  granular_event_type,
+  plan_id,
+  country,
+  country_name,
+  provider,
+  plan_amount,
+  billing_scheme,
+  plan_currency,
+  plan_interval,
+  plan_interval_count,
+  product_id,
+  product_name,
+  pricing_plan,
+  json_promotion_codes
--- a/sql/moz-fx-data-shared-prod/hubs_derived/subscription_events_v1/metadata.yaml
+++ b/sql/moz-fx-data-shared-prod/hubs_derived/subscription_events_v1/metadata.yaml
@ -0,0 +1,19 @@
+friendly_name: Hubs Subscription Events
+description: >
+  Aggregated count of Hubs subscription start/end events.
+owners:
+  - srose@mozilla.com
+labels:
+  application: hubs
+  schedule: daily
+scheduling:
+  dag_name: bqetl_subplat
+  # Delay aggregates by 8 days, to ensure data is complete. Upstream tables are
+  # delayed 7 days, and this needs an additional day of delay for cancel events.
+  date_partition_offset: -8
+  date_partition_parameter: date
+bigquery:
+  time_partitioning:
+    type: day
+    field: event_date
+    require_partition_filter: false
--- a/sql/moz-fx-data-shared-prod/hubs_derived/subscription_events_v1/query.sql
+++ b/sql/moz-fx-data-shared-prod/hubs_derived/subscription_events_v1/query.sql
@ -0,0 +1,6 @@
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod`.hubs_derived.subscription_events_live
+WHERE
+  IF(CAST(@date AS DATE) IS NULL, event_date < CURRENT_DATE - 8, event_date = @date)
--- a/sql/moz-fx-data-shared-prod/hubs_derived/subscription_events_v1/schema.yaml
+++ b/sql/moz-fx-data-shared-prod/hubs_derived/subscription_events_v1/schema.yaml
@ -0,0 +1,52 @@
+fields:
+- name: event_date
+  type: DATE
+  mode: NULLABLE
+- name: event_type
+  type: STRING
+  mode: NULLABLE
+- name: granular_event_type
+  type: STRING
+  mode: NULLABLE
+- name: plan_id
+  type: STRING
+  mode: NULLABLE
+- name: country
+  type: STRING
+  mode: NULLABLE
+- name: country_name
+  type: STRING
+  mode: NULLABLE
+- name: provider
+  type: STRING
+  mode: NULLABLE
+- name: plan_amount
+  type: INTEGER
+  mode: NULLABLE
+- name: billing_scheme
+  type: STRING
+  mode: NULLABLE
+- name: plan_currency
+  type: STRING
+  mode: NULLABLE
+- name: plan_interval
+  type: STRING
+  mode: NULLABLE
+- name: plan_interval_count
+  type: INTEGER
+  mode: NULLABLE
+- name: product_id
+  type: STRING
+  mode: NULLABLE
+- name: product_name
+  type: STRING
+  mode: NULLABLE
+- name: pricing_plan
+  type: STRING
+  mode: NULLABLE
+- name: promotion_codes
+  type: STRING
+  mode: REPEATED
+- name: count
+  type: INTEGER
+  mode: NULLABLE
--- a/sql/moz-fx-data-shared-prod/hubs_derived/subscriptions_v1/metadata.yaml
+++ b/sql/moz-fx-data-shared-prod/hubs_derived/subscriptions_v1/metadata.yaml
@ -0,0 +1,15 @@
+friendly_name: All Hubs Subscriptions
+description: >
+  Hubs subscriptions.
+owners:
+  - srose@mozilla.com
+labels:
+  application: hubs
+  schedule: daily
+scheduling:
+  dag_name: bqetl_subplat
+  # destination is the whole table, not a single partition,
+  # so don't use date_partition_parameter
+  date_partition_parameter: null
+  depends_on_fivetran:
+    - task_id: fivetran_stripe
--- a/sql/moz-fx-data-shared-prod/hubs_derived/subscriptions_v1/query.sql
+++ b/sql/moz-fx-data-shared-prod/hubs_derived/subscriptions_v1/query.sql
@ -0,0 +1,163 @@
+WITH standardized_country AS (
+  SELECT
+    raw_country AS country,
+    standardized_country AS country_name,
+  FROM
+    `moz-fx-data-shared-prod`.static.third_party_standardized_country_names
+),
+stripe_subscriptions_history AS (
+  SELECT
+    *,
+    CONCAT(
+      subscription_id,
+      COALESCE(
+        CONCAT(
+          "-",
+          NULLIF(ROW_NUMBER() OVER (PARTITION BY subscription_id ORDER BY valid_from), 1)
+        ),
+        ""
+      )
+    ) AS subscription_sequence_id
+  FROM
+    `moz-fx-data-shared-prod`.subscription_platform.stripe_subscriptions_history
+  WHERE
+    -- Only include the current history records and the last history records for previous plans.
+    (valid_to IS NULL OR plan_ended_at IS NOT NULL)
+    AND status NOT IN ("incomplete", "incomplete_expired")
+),
+hubs_subscriptions AS (
+  SELECT
+    customer_id,
+    subscription_sequence_id AS subscription_id,
+    IF(
+      subscription_sequence_id != subscription_id,
+      subscription_id,
+      NULL
+    ) AS original_subscription_id,
+    plan_id,
+    status,
+    synced_at AS event_timestamp,
+    IF(
+      (trial_end > TIMESTAMP(CURRENT_DATE) OR ended_at <= trial_end),
+      NULL,
+      COALESCE(plan_started_at, subscription_start_date)
+    ) AS subscription_start_date,
+    --first subscription start date associated with the subscription id
+    IF(
+      (trial_end > TIMESTAMP(CURRENT_DATE) OR ended_at <= trial_end),
+      NULL,
+      subscription_start_date
+    ) AS original_subscription_start_date,
+    IF(plan_started_at IS NOT NULL, "Plan Change", NULL) AS subscription_start_reason,
+    created,
+    trial_start,
+    trial_end,
+    canceled_at,
+    canceled_for_customer_at,
+    cancel_at,
+    cancel_at_period_end,
+    COALESCE(plan_ended_at, IF(ended_at < TIMESTAMP(CURRENT_DATE), ended_at, NULL)) AS ended_at,
+    IF(plan_ended_at IS NOT NULL, "Plan Change", NULL) AS ended_reason,
+    fxa_uid,
+    country,
+    country_name,
+    provider,
+    plan_amount,
+    billing_scheme,
+    plan_currency,
+    plan_interval,
+    plan_interval_count,
+    plan_interval_timezone,
+    product_id,
+    product_name,
+    CONCAT(
+      plan_interval_count,
+      "-",
+      plan_interval,
+      "-",
+      plan_currency,
+      "-",
+      (plan_amount / 100)
+    ) AS pricing_plan,
+    -- Stripe billing grace period is 7 day and Paypal is billed by Stripe
+    INTERVAL 7 DAY AS billing_grace_period,
+    promotion_codes,
+    promotion_discounts_amount,
+  FROM
+    stripe_subscriptions_history
+  LEFT JOIN
+    standardized_country
+  USING
+    (country)
+  WHERE
+    "managed-hubs" IN UNNEST(stripe_subscriptions_history.product_capabilities)
+    OR "managed-hubs" IN UNNEST(stripe_subscriptions_history.plan_capabilities)
+),
+hubs_subscriptions_with_end_date AS (
+  SELECT
+    *,
+    IF(
+      customer_id IS NOT NULL,
+      MIN(subscription_start_date) OVER (PARTITION BY customer_id),
+      subscription_start_date
+    ) AS customer_start_date,
+    COALESCE(ended_at, TIMESTAMP(CURRENT_DATE)) AS end_date,
+  FROM
+    hubs_subscriptions
+)
+SELECT
+  * REPLACE (
+    CASE
+      WHEN subscription_start_date IS NULL
+        THEN NULL
+      WHEN subscription_start_reason IS NOT NULL
+        THEN subscription_start_reason
+      WHEN trial_start IS NOT NULL
+        THEN "Converted Trial"
+      WHEN DATE(subscription_start_date) = DATE(customer_start_date)
+        THEN "New"
+      ELSE "Resurrected"
+    END AS subscription_start_reason,
+    CASE
+      WHEN ended_at IS NULL
+        THEN NULL
+      WHEN ended_reason IS NOT NULL
+        THEN ended_reason
+      WHEN canceled_for_customer_at IS NOT NULL
+        OR cancel_at_period_end
+        THEN "Cancelled by Customer"
+      ELSE "Payment Failed"
+    END AS ended_reason
+  ),
+  mozfun.norm.diff_months(
+    start => DATETIME(subscription_start_date, plan_interval_timezone),
+    `end` => DATETIME(end_date, plan_interval_timezone),
+    grace_period => billing_grace_period,
+    inclusive => FALSE
+  ) AS months_retained,
+  mozfun.norm.diff_months(
+    start => DATETIME(
+      COALESCE(original_subscription_start_date, subscription_start_date),
+      plan_interval_timezone
+    ),
+    `end` => DATETIME(end_date, plan_interval_timezone),
+    grace_period => billing_grace_period,
+    inclusive => FALSE
+  ) AS original_subscription_months_retained,
+  mozfun.norm.diff_months(
+    start => DATETIME(subscription_start_date, plan_interval_timezone),
+    `end` => DATETIME(TIMESTAMP(CURRENT_DATE), plan_interval_timezone),
+    grace_period => billing_grace_period,
+    inclusive => FALSE
+  ) AS current_months_since_subscription_start,
+  mozfun.norm.diff_months(
+    start => DATETIME(
+      COALESCE(original_subscription_start_date, subscription_start_date),
+      plan_interval_timezone
+    ),
+    `end` => DATETIME(TIMESTAMP(CURRENT_DATE), plan_interval_timezone),
+    grace_period => billing_grace_period,
+    inclusive => FALSE
+  ) AS current_months_since_original_subscription_start,
+FROM
+  hubs_subscriptions_with_end_date
--- a/sql/moz-fx-data-shared-prod/hubs_derived/subscriptions_v1/schema.yaml
+++ b/sql/moz-fx-data-shared-prod/hubs_derived/subscriptions_v1/schema.yaml
@ -0,0 +1,121 @@
+fields:
+- name: customer_id
+  type: STRING
+  mode: NULLABLE
+- name: subscription_id
+  type: STRING
+  mode: NULLABLE
+- name: original_subscription_id
+  type: STRING
+  mode: NULLABLE
+- name: plan_id
+  type: STRING
+  mode: NULLABLE
+- name: status
+  type: STRING
+  mode: NULLABLE
+- name: event_timestamp
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: customer_start_date
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: subscription_start_date
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: original_subscription_start_date
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: subscription_start_reason
+  type: STRING
+  mode: NULLABLE
+- name: created
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: trial_start
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: trial_end
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: canceled_at
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: canceled_for_customer_at
+  type: STRING
+  mode: NULLABLE
+- name: cancel_at
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: cancel_at_period_end
+  type: BOOLEAN
+  mode: NULLABLE
+- name: ended_at
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: ended_reason
+  type: STRING
+  mode: NULLABLE
+- name: end_date
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: fxa_uid
+  type: STRING
+  mode: NULLABLE
+- name: country
+  type: STRING
+  mode: NULLABLE
+- name: country_name
+  type: STRING
+  mode: NULLABLE
+- name: provider
+  type: STRING
+  mode: NULLABLE
+- name: plan_amount
+  type: INTEGER
+  mode: NULLABLE
+- name: billing_scheme
+  type: STRING
+  mode: NULLABLE
+- name: plan_currency
+  type: STRING
+  mode: NULLABLE
+- name: plan_interval
+  type: STRING
+  mode: NULLABLE
+- name: plan_interval_count
+  type: INTEGER
+  mode: NULLABLE
+- name: plan_interval_timezone
+  type: STRING
+  mode: NULLABLE
+- name: product_id
+  type: STRING
+  mode: NULLABLE
+- name: product_name
+  type: STRING
+  mode: NULLABLE
+- name: pricing_plan
+  type: STRING
+  mode: NULLABLE
+- name: billing_grace_period
+  type: INTERVAL
+  mode: NULLABLE
+- name: promotion_codes
+  type: STRING
+  mode: REPEATED
+- name: promotion_discounts_amount
+  type: INTEGER
+  mode: NULLABLE
+- name: months_retained
+  type: INTEGER
+  mode: NULLABLE
+- name: original_subscription_months_retained
+  type: INTEGER
+  mode: NULLABLE
+- name: current_months_since_subscription_start
+  type: INTEGER
+  mode: NULLABLE
+- name: current_months_since_original_subscription_start
+  type: INTEGER
+  mode: NULLABLE