DS-2642 - Import stripe itemized tax report (#3999)

2023-07-10 17:22:18 -07:00 · 2023-07-10 17:22:18 -07:00 · 6137048eeb
--- a/bigquery_etl/stripe/tax.transactions.itemized.1.schema.json
+++ b/bigquery_etl/stripe/tax.transactions.itemized.1.schema.json
@ -0,0 +1,154 @@
+[
+  {
+    "name": "id",
+    "type": "STRING"
+  },
+  {
+    "name": "type",
+    "type": "STRING"
+  },
+  {
+    "name": "currency",
+    "type": "STRING"
+  },
+  {
+    "name": "transaction_date_utc",
+    "type": "TIMESTAMP"
+  },
+  {
+    "name": "tax_date_utc",
+    "type": "TIMESTAMP"
+  },
+  {
+    "name": "refund_id",
+    "type": "STRING"
+  },
+  {
+    "name": "line_item_id",
+    "type": "STRING"
+  },
+  {
+    "name": "subtotal",
+    "type": "NUMERIC"
+  },
+  {
+    "name": "total",
+    "type": "NUMERIC"
+  },
+  {
+    "name": "tax_behavior",
+    "type": "STRING"
+  },
+  {
+    "name": "tax_code",
+    "type": "STRING"
+  },
+  {
+    "name": "jurisdiction_name",
+    "type": "STRING"
+  },
+  {
+    "name": "jurisdiction_level",
+    "type": "STRING"
+  },
+  {
+    "name": "country_code",
+    "type": "STRING"
+  },
+  {
+    "name": "state_code",
+    "type": "STRING"
+  },
+  {
+    "name": "taxability",
+    "type": "STRING"
+  },
+  {
+    "name": "taxability_reason",
+    "type": "STRING"
+  },
+  {
+    "name": "tax_rate",
+    "type": "NUMERIC"
+  },
+  {
+    "name": "quantity",
+    "type": "NUMERIC"
+  },
+  {
+    "name": "payment_intent_id",
+    "type": "STRING"
+  },
+  {
+    "name": "customer_tax_id",
+    "type": "STRING"
+  },
+  {
+    "name": "destination_resolved_address_line1",
+    "type": "STRING"
+  },
+  {
+    "name": "destination_resolved_address_line2",
+    "type": "STRING"
+  },
+  {
+    "name": "destination_resolved_address_city",
+    "type": "STRING"
+  },
+  {
+    "name": "destination_resolved_address_state",
+    "type": "STRING"
+  },
+  {
+    "name": "destination_resolved_address_postal_code",
+    "type": "STRING"
+  },
+  {
+    "name": "destination_resolved_address_country",
+    "type": "STRING"
+  },
+  {
+    "name": "tax_name",
+    "type": "STRING"
+  },
+  {
+    "name": "tax_type",
+    "type": "STRING"
+  },
+  {
+    "name": "taxable_amount",
+    "type": "NUMERIC"
+  },
+  {
+    "name": "non_taxable_amount",
+    "type": "NUMERIC"
+  },
+  {
+    "name": "tax_amount",
+    "type": "NUMERIC"
+  },
+  {
+    "name": "filing_currency",
+    "type": "STRING"
+  },
+  {
+    "name": "filing_exchange_rate",
+    "type": "NUMERIC"
+  },
+  {
+    "name": "filing_total",
+    "type": "NUMERIC"
+  },
+  {
+    "name": "filing_non_taxable_amount",
+    "type": "NUMERIC"
+  },
+  {
+    "name": "filing_taxable_amount",
+    "type": "NUMERIC"
+  },
+  {
+    "name": "filing_tax_amount",
+    "type": "NUMERIC"
+  }
+]
--- a/dags/bqetl_subplat.py
+++ b/dags/bqetl_subplat.py
@ -799,6 +799,27 @@ with DAG(
        email_on_retry=False,
    )

+    stripe_external__itemized_tax_transactions__v1 = gke_command(
+        task_id="stripe_external__itemized_tax_transactions__v1",
+        command=[
+            "python",
+            "sql/moz-fx-data-shared-prod/stripe_external/itemized_tax_transactions_v1/query.py",
+        ]
+        + [
+            "--date={{ ds }}",
+            "--api-key={{ var.value.stripe_api_key }}",
+            "--report-type=tax.transactions.itemized.1",
+            "--table=moz-fx-data-shared-prod.stripe_external.itemized_tax_transactions_v1",
+            "--time-partitioning-field=transaction_date_utc",
+        ],
+        docker_image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest",
+        owner="srose@mozilla.com",
+        email=["srose@mozilla.com", "telemetry-alerts@mozilla.com"],
+        retry_delay=datetime.timedelta(seconds=1800),
+        retries=47,
+        email_on_retry=False,
+    )
+
    stripe_external__plan__v1 = bigquery_etl_query(
        task_id="stripe_external__plan__v1",
        destination_table="plan_v1",
--- a/sql/moz-fx-data-shared-prod/stripe_external/itemized_tax_transactions_v1/metadata.yaml
+++ b/sql/moz-fx-data-shared-prod/stripe_external/itemized_tax_transactions_v1/metadata.yaml
@ -0,0 +1,29 @@
+---
+friendly_name: Stripe itemized tax transactions report
+description: >
+  Import itemized tax transactions report from the Stripe API.
+owners:
+  - srose@mozilla.com
+labels:
+  incremental: true
+  schedule: daily
+scheduling:
+  dag_name: bqetl_subplat
+  arguments:
+    - --date={{ ds }}
+    - --api-key={{ var.value.stripe_api_key }}
+    - --report-type=tax.transactions.itemized.1
+    - --table=moz-fx-data-shared-prod.stripe_external.itemized_tax_transactions_v1
+    - --time-partitioning-field=transaction_date_utc
+  # report data isn't generally ready when the dag starts, so retry for a while
+  retry_delay: 30m
+  retries: 47
+  email_on_retry: false
+bigquery:
+  time_partitioning:
+    type: day
+    field: transaction_date_utc
+    require_partition_filter: false
+  clustering:
+    fields:
+      - transaction_date_utc
--- a/sql/moz-fx-data-shared-prod/stripe_external/itemized_tax_transactions_v1/query.py
+++ b/sql/moz-fx-data-shared-prod/stripe_external/itemized_tax_transactions_v1/query.py
@ -0,0 +1,6 @@
+"""Import itemized tax transactions report from the Stripe API."""
+
+from bigquery_etl.stripe import stripe_import
+
+if __name__ == "__main__":
+    stripe_import()
--- a/sql/moz-fx-data-shared-prod/stripe_external/itemized_tax_transactions_v1/schema.yaml
+++ b/sql/moz-fx-data-shared-prod/stripe_external/itemized_tax_transactions_v1/schema.yaml
@ -0,0 +1,116 @@
+---
+fields:
+- name: id
+  type: STRING
+  mode: NULLABLE
+- name: type
+  type: STRING
+  mode: NULLABLE
+- name: currency
+  type: STRING
+  mode: NULLABLE
+- name: transaction_date_utc
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: tax_date_utc
+  type: TIMESTAMP
+  mode: NULLABLE
+- name: refund_id
+  type: STRING
+  mode: NULLABLE
+- name: line_item_id
+  type: STRING
+  mode: NULLABLE
+- name: subtotal
+  type: NUMERIC
+  mode: NULLABLE
+- name: total
+  type: NUMERIC
+  mode: NULLABLE
+- name: tax_behavior
+  type: STRING
+  mode: NULLABLE
+- name: tax_code
+  type: STRING
+  mode: NULLABLE
+- name: jurisdiction_name
+  type: STRING
+  mode: NULLABLE
+- name: jurisdiction_level
+  type: STRING
+  mode: NULLABLE
+- name: country_code
+  type: STRING
+  mode: NULLABLE
+- name: state_code
+  type: STRING
+  mode: NULLABLE
+- name: taxability
+  type: STRING
+  mode: NULLABLE
+- name: taxability_reason
+  type: STRING
+  mode: NULLABLE
+- name: tax_rate
+  type: NUMERIC
+  mode: NULLABLE
+- name: quantity
+  type: NUMERIC
+  mode: NULLABLE
+- name: payment_intent_id
+  type: STRING
+  mode: NULLABLE
+- name: customer_tax_id
+  type: STRING
+  mode: NULLABLE
+- name: destination_resolved_address_line1
+  type: STRING
+  mode: NULLABLE
+- name: destination_resolved_address_line2
+  type: STRING
+  mode: NULLABLE
+- name: destination_resolved_address_city
+  type: STRING
+  mode: NULLABLE
+- name: destination_resolved_address_state
+  type: STRING
+  mode: NULLABLE
+- name: destination_resolved_address_postal_code
+  type: STRING
+  mode: NULLABLE
+- name: destination_resolved_address_country
+  type: STRING
+  mode: NULLABLE
+- name: tax_name
+  type: STRING
+  mode: NULLABLE
+- name: tax_type
+  type: STRING
+  mode: NULLABLE
+- name: taxable_amount
+  type: NUMERIC
+  mode: NULLABLE
+- name: non_taxable_amount
+  type: NUMERIC
+  mode: NULLABLE
+- name: tax_amount
+  type: NUMERIC
+  mode: NULLABLE
+- name: filing_currency
+  type: STRING
+  mode: NULLABLE
+- name: filing_exchange_rate
+  type: NUMERIC
+  mode: NULLABLE
+- name: filing_total
+  type: NUMERIC
+  mode: NULLABLE
+- name: filing_non_taxable_amount
+  type: NUMERIC
+  mode: NULLABLE
+- name: filing_taxable_amount
+  type: NUMERIC
+  mode: NULLABLE
+- name: filing_tax_amount
+  type: NUMERIC
+  mode: NULLABLE