Add `stripe_products_v1` and `stripe_plans_v1` ETLs (DENG-1448). (#4234)

bqetl_project.yaml

@@ -98,9 +98,10 @@ dry_run:
   - sql/moz-fx-data-shared-prod/subscription_platform_derived/nonprod_apple_subscriptions_v1/query.sql
   - sql/moz-fx-data-shared-prod/subscription_platform_derived/nonprod_google_subscriptions_v1/query.sql
   - sql/moz-fx-data-shared-prod/subscription_platform_derived/nonprod_stripe_subscriptions_history_v1/query.sql
-  - sql/moz-fx-data-shared-prod/subscription_platform_derived/services_v1/query.sql
   - sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_customers_revised_changelog_v1/query.sql
   - sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_logical_subscriptions_history_v1/query.sql
+  - sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_plans_v1/query.sql
+  - sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_products_v1/query.sql
   - sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_subscriptions_revised_changelog_v1/query.sql
   - sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_subscriptions_history_v1/query.sql
   - sql/moz-fx-data-shared-prod/stripe/itemized_payout_reconciliation/view.sql

dags/bqetl_subplat.py

@@ -1096,6 +1096,30 @@ with DAG(
         task_concurrency=1,
     )
 
+    subscription_platform_derived__stripe_plans__v1 = bigquery_etl_query(
+        task_id="subscription_platform_derived__stripe_plans__v1",
+        destination_table="stripe_plans_v1",
+        dataset_id="subscription_platform_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="srose@mozilla.com",
+        email=["srose@mozilla.com", "telemetry-alerts@mozilla.com"],
+        date_partition_parameter=None,
+        depends_on_past=False,
+        task_concurrency=1,
+    )
+
+    subscription_platform_derived__stripe_products__v1 = bigquery_etl_query(
+        task_id="subscription_platform_derived__stripe_products__v1",
+        destination_table="stripe_products_v1",
+        dataset_id="subscription_platform_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="srose@mozilla.com",
+        email=["srose@mozilla.com", "telemetry-alerts@mozilla.com"],
+        date_partition_parameter=None,
+        depends_on_past=False,
+        task_concurrency=1,
+    )
+
     subscription_platform_derived__stripe_subscriptions__v1 = bigquery_etl_query(
         task_id="subscription_platform_derived__stripe_subscriptions__v1",
         destination_table="stripe_subscriptions_v1",
@@ -1558,10 +1582,12 @@ with DAG(
         mozilla_vpn_derived__guardian_apple_events__v1
     )
 
-    subscription_platform_derived__services__v1.set_upstream(stripe_external__plan__v1)
+    subscription_platform_derived__services__v1.set_upstream(
+        subscription_platform_derived__stripe_plans__v1
+    )
 
     subscription_platform_derived__services__v1.set_upstream(
-        stripe_external__product__v1
+        subscription_platform_derived__stripe_products__v1
     )
 
     subscription_platform_derived__stripe_customers_history__v1.set_upstream(
@@ -1600,6 +1626,14 @@ with DAG(
         subscription_platform_derived__stripe_subscriptions_history__v2
     )
 
+    subscription_platform_derived__stripe_plans__v1.set_upstream(
+        stripe_external__plan__v1
+    )
+
+    subscription_platform_derived__stripe_products__v1.set_upstream(
+        stripe_external__product__v1
+    )
+
     subscription_platform_derived__stripe_subscriptions__v1.set_upstream(
         subscription_platform_derived__stripe_subscriptions_history__v1
     )
@@ -1660,18 +1694,18 @@ with DAG(
         stripe_external__invoice_line_item__v1
     )
 
-    subscription_platform_derived__stripe_subscriptions_revised_changelog__v1.set_upstream(
-        stripe_external__plan__v1
-    )
-
-    subscription_platform_derived__stripe_subscriptions_revised_changelog__v1.set_upstream(
-        stripe_external__product__v1
-    )
-
     subscription_platform_derived__stripe_subscriptions_revised_changelog__v1.set_upstream(
         stripe_external__subscriptions_changelog__v1
     )
 
+    subscription_platform_derived__stripe_subscriptions_revised_changelog__v1.set_upstream(
+        subscription_platform_derived__stripe_plans__v1
+    )
+
+    subscription_platform_derived__stripe_subscriptions_revised_changelog__v1.set_upstream(
+        subscription_platform_derived__stripe_products__v1
+    )
+
     subscription_platform_derived__subplat_attribution_impressions__v1.set_upstream(
         subscription_platform_derived__services__v1
     )

sql/moz-fx-data-shared-prod/subscription_platform/stripe_plans/view.sql

@@ -0,0 +1,7 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.subscription_platform.stripe_plans`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod.subscription_platform_derived.stripe_plans_v1`

sql/moz-fx-data-shared-prod/subscription_platform/stripe_products/view.sql

@@ -0,0 +1,7 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.subscription_platform.stripe_products`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod.subscription_platform_derived.stripe_products_v1`

sql/moz-fx-data-shared-prod/subscription_platform_derived/services_v1/query.sql

@@ -66,17 +66,17 @@ WITH services AS (
 stripe_products AS (
   SELECT
     id,
-    PARSE_JSON(metadata) AS metadata
+    metadata
   FROM
-    `moz-fx-data-shared-prod.stripe_external.product_v1`
+    `moz-fx-data-shared-prod.subscription_platform_derived.stripe_products_v1`
 ),
 stripe_plans AS (
   SELECT
     id,
-    PARSE_JSON(metadata) AS metadata,
-    product_id
+    product_id,
+    metadata
   FROM
-    `moz-fx-data-shared-prod.stripe_external.plan_v1`
+    `moz-fx-data-shared-prod.subscription_platform_derived.stripe_plans_v1`
 ),
 service_stripe_product_ids AS (
   SELECT

sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_plans_v1/metadata.yaml

@@ -0,0 +1,15 @@
+friendly_name: Stripe plans
+description: |-
+  Stripe plans.
+owners:
+- srose@mozilla.com
+labels:
+  incremental: false
+scheduling:
+  dag_name: bqetl_subplat
+  # The whole table is overwritten every time, not a specific date partition.
+  date_partition_parameter: null
+bigquery:
+  time_partitioning: null
+  clustering: null
+references: {}

sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_plans_v1/query.sql

@@ -0,0 +1,19 @@
+SELECT
+  id,
+  product_id,
+  created,
+  is_deleted,
+  active,
+  aggregate_usage,
+  amount,
+  billing_scheme,
+  currency,
+  `interval`,
+  interval_count,
+  PARSE_JSON(metadata) AS metadata,
+  nickname,
+  tiers_mode,
+  trial_period_days,
+  usage_type,
+FROM
+  `moz-fx-data-shared-prod.stripe_external.plan_v1`

sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_plans_v1/schema.yaml

@@ -0,0 +1,83 @@
+fields:
+- name: id
+  type: STRING
+  mode: NULLABLE
+  description: Unique identifier for the plan.
+- name: product_id
+  type: STRING
+  mode: NULLABLE
+  description: ID of the product whose pricing this plan determines.
+- name: created
+  type: TIMESTAMP
+  mode: NULLABLE
+  description: Time at which the plan was created.
+- name: is_deleted
+  type: BOOLEAN
+  mode: NULLABLE
+  description: Whether the plan is deleted.
+- name: active
+  type: BOOLEAN
+  mode: NULLABLE
+  description: Whether the plan can be used for new purchases.
+- name: aggregate_usage
+  type: STRING
+  mode: NULLABLE
+  description: Specifies a usage aggregation strategy for plans of `usage_type` =
+    "metered". Allowed values are "sum" for summing up all usage during a period,
+    "last_during_period" for using the last usage record reported within a period,
+    "last_ever" for using the last usage record ever (across period bounds) or "max"
+    which uses the usage record with the maximum reported usage during a period.
+- name: amount
+  type: INTEGER
+  mode: NULLABLE
+  description: The unit amount in cents to be charged, represented as a whole integer
+    if possible. Only set if `billing_scheme` = "per_unit".
+- name: billing_scheme
+  type: STRING
+  mode: NULLABLE
+  description: Describes how to compute the price per period. Either "per_unit" or
+    "tiered". "per_unit" indicates that the fixed amount (specified in `amount`) will
+    be charged per unit in `quantity` (for plans with `usage_type` = "licensed"),
+    or per unit of total usage (for plans with `usage_type` = "metered"). "tiered"
+    indicates that the unit pricing will be computed using a tiering strategy as defined
+    using the `tiers` and `tiers_mode` attributes.
+- name: currency
+  type: STRING
+  mode: NULLABLE
+  description: Three-letter ISO currency code, in lowercase.
+- name: interval
+  type: STRING
+  mode: NULLABLE
+  description: The frequency at which a subscription is billed. One of "day", "week",
+    "month", or "year".
+- name: interval_count
+  type: INTEGER
+  mode: NULLABLE
+  description: The number of intervals (specified in the `interval` attribute) between
+    subscription billings.
+- name: metadata
+  type: JSON
+  mode: NULLABLE
+  description: Set of key-value pairs attached to the plan, stored as a JSON object.
+- name: nickname
+  type: STRING
+  mode: NULLABLE
+  description: A brief description of the plan, hidden from customers.
+- name: tiers_mode
+  type: STRING
+  mode: NULLABLE
+  description: Defines if the tiering price should be "graduated" or "volume". In
+    volume-based tiering, the maximum quantity within a period determines the per
+    unit price. In graduated tiering, pricing can change as the quantity grows.
+- name: trial_period_days
+  type: INTEGER
+  mode: NULLABLE
+  description: Default number of trial days when subscribing a customer to this plan
+    using `trial_from_plan=true`.
+- name: usage_type
+  type: STRING
+  mode: NULLABLE
+  description: Configures how the quantity per period should be determined. Can be
+    either "metered" or "licensed". "licensed" automatically bills the `quantity`
+    set when adding it to a subscription. "metered" aggregates the total usage based
+    on usage records. Defaults to "licensed".

sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_products_v1/metadata.yaml

@@ -0,0 +1,15 @@
+friendly_name: Stripe products
+description: |-
+  Stripe products.
+owners:
+- srose@mozilla.com
+labels:
+  incremental: false
+scheduling:
+  dag_name: bqetl_subplat
+  # The whole table is overwritten every time, not a specific date partition.
+  date_partition_parameter: null
+bigquery:
+  time_partitioning: null
+  clustering: null
+references: {}

sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_products_v1/query.sql

@@ -0,0 +1,12 @@
+SELECT
+  id,
+  created,
+  updated,
+  is_deleted,
+  active,
+  description,
+  PARSE_JSON(metadata) AS metadata,
+  name,
+  statement_descriptor,
+FROM
+  `moz-fx-data-shared-prod.stripe_external.product_v1`

sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_products_v1/schema.yaml

@@ -0,0 +1,38 @@
+fields:
+- name: id
+  type: STRING
+  mode: NULLABLE
+  description: Unique identifier for the product.
+- name: created
+  type: TIMESTAMP
+  mode: NULLABLE
+  description: Time at which the product was created.
+- name: updated
+  type: TIMESTAMP
+  mode: NULLABLE
+  description: Time at which the product was last updated.
+- name: is_deleted
+  type: BOOLEAN
+  mode: NULLABLE
+  description: Whether the product is deleted.
+- name: active
+  type: BOOLEAN
+  mode: NULLABLE
+  description: Whether the product is currently available for purchase.
+- name: description
+  type: STRING
+  mode: NULLABLE
+  description: The product's description, meant to be displayable to the customer.
+- name: metadata
+  type: JSON
+  mode: NULLABLE
+  description: Set of key-value pairs attached to the product, stored as a JSON object.
+- name: name
+  type: STRING
+  mode: NULLABLE
+  description: The product's name, meant to be displayable to the customer.
+- name: statement_descriptor
+  type: STRING
+  mode: NULLABLE
+  description: Extra information about a product which will appear on your customer's
+    credit card statement.

sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_subscriptions_revised_changelog_v1/query.sql

@@ -208,13 +208,13 @@ questionable_subscription_plans_history AS (
       plans.currency,
       plans.`interval`,
       plans.interval_count,
-      PARSE_JSON(plans.metadata) AS metadata,
+      plans.metadata,
       plans.nickname,
       STRUCT(
         plans.product_id AS id,
         products.created,
         products.description,
-        PARSE_JSON(products.metadata) AS metadata,
+        products.metadata,
         products.name,
         products.statement_descriptor,
         products.updated
@@ -233,11 +233,11 @@ questionable_subscription_plans_history AS (
   FROM
     questionable_subscription_plan_changes AS plan_changes
   LEFT JOIN
-    `moz-fx-data-shared-prod`.stripe_external.plan_v1 AS plans
+    `moz-fx-data-shared-prod`.subscription_platform_derived.stripe_plans_v1 AS plans
   ON
     plan_changes.plan_id = plans.id
   LEFT JOIN
-    `moz-fx-data-shared-prod`.stripe_external.product_v1 AS products
+    `moz-fx-data-shared-prod`.subscription_platform_derived.stripe_products_v1 AS products
   ON
     plans.product_id = products.id
   WINDOW