Add fivetran_costs (#3509)

* Add fivetran_costs

* Add fivetran_costs - adding schemas to all tables and adressing other suggestions

* Add fivetran_costs - renaming tables, adding to the schema

* Add fivetran_costs - adding fivetran-dev

* Add fivetran_costs - adding tests

* Add fivetran_costs - adding tests

* Add fivetran_costs - adding tests

* Add fivetran_costs - adding tests

* implementing suggestions

* rerun dag creation

bigquery_etl/dryrun.py

@@ -83,6 +83,9 @@ SKIP = {
     "sql/moz-fx-data-shared-prod/firefox_accounts_derived/docker_fxa_admin_server_sanitized_v1/query.sql",  # noqa E501
     "sql/moz-fx-data-shared-prod/firefox_accounts_derived/docker_fxa_customs_sanitized_v1/init.sql",  # noqa E501
     "sql/moz-fx-data-shared-prod/firefox_accounts_derived/docker_fxa_customs_sanitized_v1/query.sql",  # noqa E501
+    "sql/moz-fx-data-shared-prod/fivetran_costs_derived/destinations_v1/query.sql",
+    "sql/moz-fx-data-shared-prod/fivetran_costs_derived/incremental_mar_v1/query.sql",
+    "sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_costs_v1/query.sql",
     "sql/moz-fx-data-shared-prod/regrets_reporter/regrets_reporter_update/view.sql",
     "sql/moz-fx-data-shared-prod/revenue_derived/client_ltv_v1/query.sql",
     "sql/moz-fx-data-shared-prod/monitoring/payload_bytes_decoded_structured/view.sql",

dags.yaml

@@ -957,3 +957,22 @@ bqetl_campaign_cost_breakdowns:
   tags:
     - impact/tier_2
     - repo/bigquery-etl
+bqetl_fivetran_costs:
+  default_args:
+    depends_on_past: false
+    email:
+      - telemetry-alerts@mozilla.com
+      - lschiestl@mozilla.com
+    email_on_failure: true
+    email_on_retry: true
+    end_date: null
+    owner: lschiestl@mozilla.com
+    retries: 2
+    retry_delay: 30m
+    start_date: '2023-01-18'
+  description: |
+    Derived tables for analyzing the Fivetran Costs. Data coming from Fivetran.
+  repo: bigquery-etl
+  schedule_interval: 0 5 * * *
+  tags:
+    - impact/tier_3

dags/bqetl_fivetran_costs.py

@@ -0,0 +1,159 @@
+# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py
+
+from airflow import DAG
+from airflow.sensors.external_task import ExternalTaskMarker
+from airflow.sensors.external_task import ExternalTaskSensor
+from airflow.utils.task_group import TaskGroup
+import datetime
+from utils.constants import ALLOWED_STATES, FAILED_STATES
+from utils.gcp import bigquery_etl_query, gke_command
+
+from fivetran_provider.operators.fivetran import FivetranOperator
+from fivetran_provider.sensors.fivetran import FivetranSensor
+from utils.callbacks import retry_tasks_callback
+
+docs = """
+### bqetl_fivetran_costs
+
+Built from bigquery-etl repo, [`dags/bqetl_fivetran_costs.py`](https://github.com/mozilla/bigquery-etl/blob/main/dags/bqetl_fivetran_costs.py)
+
+#### Description
+
+Derived tables for analyzing the Fivetran Costs. Data coming from Fivetran.
+
+#### Owner
+
+lschiestl@mozilla.com
+"""
+
+
+default_args = {
+    "owner": "lschiestl@mozilla.com",
+    "start_date": datetime.datetime(2023, 1, 18, 0, 0),
+    "end_date": None,
+    "email": ["telemetry-alerts@mozilla.com", "lschiestl@mozilla.com"],
+    "depends_on_past": False,
+    "retry_delay": datetime.timedelta(seconds=1800),
+    "email_on_failure": True,
+    "email_on_retry": True,
+    "retries": 2,
+}
+
+tags = ["impact/tier_3", "repo/bigquery-etl"]
+
+with DAG(
+    "bqetl_fivetran_costs",
+    default_args=default_args,
+    schedule_interval="0 5 * * *",
+    doc_md=docs,
+    tags=tags,
+) as dag:
+
+    fivetran_costs_derived__destinations__v1 = bigquery_etl_query(
+        task_id="fivetran_costs_derived__destinations__v1",
+        destination_table="destinations_v1",
+        dataset_id="fivetran_costs_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="lschiestl@mozilla.com",
+        email=["lschiestl@mozilla.com", "telemetry-alerts@mozilla.com"],
+        date_partition_parameter=None,
+        depends_on_past=False,
+        task_concurrency=1,
+    )
+
+    fivetran_costs_derived__incremental_mar__v1 = bigquery_etl_query(
+        task_id="fivetran_costs_derived__incremental_mar__v1",
+        destination_table="incremental_mar_v1",
+        dataset_id="fivetran_costs_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="lschiestl@mozilla.com",
+        email=["lschiestl@mozilla.com", "telemetry-alerts@mozilla.com"],
+        date_partition_parameter=None,
+        depends_on_past=False,
+        task_concurrency=1,
+    )
+
+    fivetran_costs_derived__monthly_connector_costs__v1 = bigquery_etl_query(
+        task_id="fivetran_costs_derived__monthly_connector_costs__v1",
+        destination_table="monthly_connector_costs_v1",
+        dataset_id="fivetran_costs_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="lschiestl@mozilla.com",
+        email=["lschiestl@mozilla.com", "telemetry-alerts@mozilla.com"],
+        date_partition_parameter=None,
+        depends_on_past=False,
+        task_concurrency=1,
+    )
+
+    fivetran_costs_derived__monthly_costs__v1 = bigquery_etl_query(
+        task_id="fivetran_costs_derived__monthly_costs__v1",
+        destination_table="monthly_costs_v1",
+        dataset_id="fivetran_costs_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="lschiestl@mozilla.com",
+        email=["lschiestl@mozilla.com", "telemetry-alerts@mozilla.com"],
+        date_partition_parameter=None,
+        depends_on_past=False,
+        task_concurrency=1,
+    )
+
+    fivetran_log_prod_sync_start = FivetranOperator(
+        connector_id="{{ var.value.fivetran_log_prod_connector_id }}",
+        task_id="fivetran_log_prod_task",
+    )
+
+    fivetran_log_prod_sync_wait = FivetranSensor(
+        connector_id="{{ var.value.fivetran_log_prod_connector_id }}",
+        task_id="fivetran_log_prod_sensor",
+        poke_interval=30,
+        xcom="{{ task_instance.xcom_pull('fivetran_log_prod_task') }}",
+        on_retry_callback=retry_tasks_callback,
+        params={"retry_tasks": ["fivetran_log_prod_task"]},
+    )
+
+    fivetran_log_prod_sync_wait.set_upstream(fivetran_log_prod_sync_start)
+
+    fivetran_costs_derived__destinations__v1.set_upstream(fivetran_log_prod_sync_wait)
+
+    fivetran_log_dev_sync_start = FivetranOperator(
+        connector_id="{{ var.value.fivetran_log_dev_connector_id }}",
+        task_id="fivetran_log_dev_task",
+    )
+
+    fivetran_log_dev_sync_wait = FivetranSensor(
+        connector_id="{{ var.value.fivetran_log_dev_connector_id }}",
+        task_id="fivetran_log_dev_sensor",
+        poke_interval=30,
+        xcom="{{ task_instance.xcom_pull('fivetran_log_dev_task') }}",
+        on_retry_callback=retry_tasks_callback,
+        params={"retry_tasks": ["fivetran_log_dev_task"]},
+    )
+
+    fivetran_log_dev_sync_wait.set_upstream(fivetran_log_dev_sync_start)
+
+    fivetran_costs_derived__destinations__v1.set_upstream(fivetran_log_dev_sync_wait)
+
+    fivetran_costs_derived__incremental_mar__v1.set_upstream(
+        fivetran_log_prod_sync_wait
+    )
+
+    fivetran_costs_derived__incremental_mar__v1.set_upstream(fivetran_log_dev_sync_wait)
+
+    fivetran_costs_derived__monthly_connector_costs__v1.set_upstream(
+        fivetran_costs_derived__destinations__v1
+    )
+
+    fivetran_costs_derived__monthly_connector_costs__v1.set_upstream(
+        fivetran_costs_derived__incremental_mar__v1
+    )
+
+    fivetran_costs_derived__monthly_connector_costs__v1.set_upstream(
+        fivetran_costs_derived__monthly_costs__v1
+    )
+    fivetran_costs_derived__monthly_connector_costs__v1.set_upstream(
+        fivetran_log_prod_sync_wait
+    )
+
+    fivetran_costs_derived__monthly_costs__v1.set_upstream(fivetran_log_prod_sync_wait)
+
+    fivetran_costs_derived__monthly_costs__v1.set_upstream(fivetran_log_dev_sync_wait)

sql/moz-fx-data-shared-prod/fivetran_costs/dataset_metadata.yaml

@@ -0,0 +1,10 @@
+friendly_name: Fivetran Costs
+description: |-
+  Analyze the costs for Fivetran connectors.
+dataset_base_acl: view
+user_facing: true
+labels: {}
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential

sql/moz-fx-data-shared-prod/fivetran_costs/monthly_connector_costs/view.sql

@@ -0,0 +1,7 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.fivetran_costs.monthly_connector_costs`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod.fivetran_costs_derived.monthly_connector_costs_v1`

sql/moz-fx-data-shared-prod/fivetran_costs_derived/dataset_metadata.yaml

@@ -0,0 +1,10 @@
+friendly_name: Fivetran Costs Derived
+description: |-
+  Derived Tables for Fivetran Costs populated from Fivetran Logs. Destinations Fivetran_Prod and Fivetran_Dev
+dataset_base_acl: derived
+user_facing: false
+labels: {}
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential

sql/moz-fx-data-shared-prod/fivetran_costs_derived/destinations_v1/metadata.yaml

@@ -0,0 +1,14 @@
+friendly_name: Destinations
+description: |-
+  Destinations Table for Fivetran Costs.
+owners:
+- lschiestl@mozilla.com
+labels: {}
+scheduling:
+  dag_name: bqetl_fivetran_costs
+  date_partition_parameter: null
+  depends_on_fivetran:
+  - task_id: fivetran_log_prod
+  - task_id: fivetran_log_dev
+bigquery: null
+references: {}

sql/moz-fx-data-shared-prod/fivetran_costs_derived/destinations_v1/query.sql

@@ -0,0 +1,17 @@
+SELECT
+  id AS destination_id,
+  name AS destination_name,
+  account_id,
+  created_at,
+  region
+FROM
+  `moz-fx-data-bq-fivetran.fivetran_log.destination`
+UNION ALL
+SELECT
+  id AS destination_id,
+  name AS destination_name,
+  account_id,
+  created_at,
+  region
+FROM
+  `dev-fivetran.fivetran_log.destination`

sql/moz-fx-data-shared-prod/fivetran_costs_derived/destinations_v1/schema.yaml

@@ -0,0 +1,26 @@
+fields:
+- mode: NULLABLE
+  name: destination_id
+  type: STRING
+  description: |
+    Unique ID of the destination warehouse.
+- mode: NULLABLE
+  name: destination_name
+  type: STRING
+  description: |
+    Name of the destination warehouse.
+- mode: NULLABLE
+  name: account_id
+  type: STRING
+  description: |
+    Foreign key referencing the Fivetran account associated with the destination.
+- mode: NULLABLE
+  name: created_at
+  type: TIMESTAMP
+  description: |
+    Timestamp of when the destination was set up.
+- mode: NULLABLE
+  name: region
+  type: STRING
+  description: |
+    Geographical region of the destination.

sql/moz-fx-data-shared-prod/fivetran_costs_derived/incremental_mar_v1/metadata.yaml

@@ -0,0 +1,14 @@
+friendly_name: Incremental MAR
+description: |-
+  Incremental MAR (monthly active rows) populated from Fivetran logs
+owners:
+- lschiestl@mozilla.com
+labels: {}
+scheduling:
+  dag_name: bqetl_fivetran_costs
+  date_partition_parameter: null
+  depends_on_fivetran:
+  - task_id: fivetran_log_prod
+  - task_id: fivetran_log_dev
+bigquery: null
+references: {}

sql/moz-fx-data-shared-prod/fivetran_costs_derived/incremental_mar_v1/query.sql

@@ -0,0 +1,32 @@
+WITH incremental_mar AS (
+  SELECT
+    measured_date,
+    destination_id,
+    connector_id,
+    table_name,
+    free_type,
+    incremental_rows
+  FROM
+    `moz-fx-data-bq-fivetran.fivetran_log.incremental_mar`
+  UNION ALL
+  SELECT
+    measured_date,
+    destination_id,
+    connector_id,
+    table_name,
+    free_type,
+    incremental_rows
+  FROM
+    `dev-fivetran.fivetran_log.incremental_mar`
+)
+SELECT
+  measured_date,
+  DATE_TRUNC(measured_date, month) AS measured_month,
+  destination_id,
+  connector_id AS connector_name,
+  table_name,
+  IF(LOWER(free_type) = "paid", incremental_rows, 0) AS paid_active_rows,
+  IF(LOWER(free_type) != "paid", incremental_rows, 0) AS free_active_rows,
+  incremental_rows AS total_active_rows
+FROM
+  incremental_mar

sql/moz-fx-data-shared-prod/fivetran_costs_derived/incremental_mar_v1/schema.yaml

@@ -0,0 +1,41 @@
+fields:
+- mode: NULLABLE
+  name: measured_date
+  type: DATE
+  description: |
+    Date of when the active row measurment was made.
+- mode: NULLABLE
+  name: measured_month
+  type: DATE
+  description: |
+    Month of when the active row measurement was made.
+- mode: NULLABLE
+  name: destination_id
+  type: STRING
+  description: |
+    Foreign key referencing the destination whose table is being measured.
+- mode: NULLABLE
+  name: connector_name
+  type: STRING
+  description: |
+    The name of the connector being measured.
+- mode: NULLABLE
+  name: table_name
+  type: STRING
+  description: |
+    The name of the table whose active rows were measured.
+- mode: NULLABLE
+  name: paid_active_rows
+  type: INTEGER
+  description: |
+    The number of new distinct primary keys on the current day synced for the connector that were paid for.
+- mode: NULLABLE
+  name: free_active_rows
+  type: INTEGER
+  description: |
+    The number of new distinct primary keys on the current day synced for the connector without any costs (initial sync, free connectors, ...)
+- mode: NULLABLE
+  name: total_active_rows
+  type: INTEGER
+  description: |
+    The number of new distinct primary keys on the current day synced for the connector.

sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_connector_costs_v1/metadata.yaml

@@ -0,0 +1,13 @@
+friendly_name: Monthly Connector Costs
+description: |-
+  Monthly Costs per Fivetran Connector
+owners:
+- lschiestl@mozilla.com
+labels: {}
+scheduling:
+  dag_name: bqetl_fivetran_costs
+  date_partition_parameter: null
+  depends_on_fivetran:
+  - task_id: fivetran_log_prod
+bigquery: null
+references: {}

sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_connector_costs_v1/query.sql

@@ -0,0 +1,96 @@
+WITH connector_mar AS (
+  SELECT
+    destination_id,
+    measured_month,
+    connector_name,
+    SUM(paid_active_rows) AS paid_active_rows,
+    SUM(free_active_rows) AS free_active_rows,
+    SUM(total_active_rows) AS total_active_rows
+  FROM
+    `moz-fx-data-shared-prod.fivetran_costs_derived.incremental_mar_v1`
+  GROUP BY
+    destination_id,
+    measured_month,
+    connector_name
+),
+destinations_total_mar AS (
+  SELECT
+    destination_id,
+    measured_month,
+    SUM(paid_active_rows) AS paid_active_rows,
+    SUM(total_active_rows) AS total_active_rows
+  FROM
+    `moz-fx-data-shared-prod.fivetran_costs_derived.incremental_mar_v1`
+  GROUP BY
+    destination_id,
+    measured_month
+),
+total_mar AS (
+  SELECT
+    measured_month,
+    SUM(paid_active_rows) AS paid_active_rows,
+  FROM
+    destinations_total_mar
+  GROUP BY
+    measured_month
+),
+monthly_connector_ratios AS (
+  SELECT
+    destination_id,
+    measured_month,
+    connector_mar.connector_name,
+    connector_mar.paid_active_rows,
+    connector_mar.free_active_rows,
+    connector_mar.total_active_rows,
+    ROUND(
+      connector_mar.paid_active_rows / NULLIF(destinations_total_mar.paid_active_rows, 0) * 100,
+      2
+    ) AS percentage_of_destination_paid_active_rows,
+    ROUND(
+      connector_mar.paid_active_rows / NULLIF(total_mar.paid_active_rows, 0) * 100,
+      2
+    ) AS percentage_of_total_active_rows
+  FROM
+    connector_mar
+  LEFT JOIN
+    destinations_total_mar
+  USING
+    (destination_id, measured_month)
+  LEFT JOIN
+    total_mar
+  USING
+    (measured_month)
+),
+monthly_connector_costs AS (
+  SELECT
+    destinations.destination_name,
+    measured_month,
+    monthly_connector_ratios.connector_name,
+    monthly_connector_ratios.paid_active_rows,
+    monthly_connector_ratios.free_active_rows,
+    monthly_connector_ratios.total_active_rows,
+    monthly_connector_ratios.percentage_of_destination_paid_active_rows,
+    monthly_connector_ratios.percentage_of_total_active_rows,
+    ROUND(
+      monthly_connector_ratios.percentage_of_destination_paid_active_rows * monthly_costs.dollars_spent,
+      2
+    ) AS cost_in_usd,
+    ROUND(
+      monthly_connector_ratios.percentage_of_destination_paid_active_rows * monthly_costs.credits_spent,
+      2
+    ) AS cost_in_credits
+  FROM
+    monthly_connector_ratios
+  LEFT JOIN
+    `moz-fx-data-shared-prod.fivetran_costs_derived.destinations_v1` AS destinations
+  USING
+    (destination_id)
+  LEFT JOIN
+    `moz-fx-data-shared-prod.fivetran_costs_derived.monthly_costs_v1` AS monthly_costs
+  USING
+    (destination_id, measured_month)
+)
+SELECT
+  *
+FROM
+  monthly_connector_costs

sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_connector_costs_v1/schema.yaml

@@ -0,0 +1,51 @@
+fields:
+- mode: NULLABLE
+  name: destination_name
+  type: STRING
+  description: |
+    Name of the Fivetran destination as it appears in the UI.
+- mode: NULLABLE
+  name: measured_month
+  type: DATE
+  description: |
+    Month of the usage measurement.
+- mode: NULLABLE
+  name: connector_name
+  type: STRING
+  description: |
+    Name of the Fivetran connector billed.
+- mode: NULLABLE
+  name: paid_active_rows
+  type: INTEGER
+  description: |
+    Number of active rows billed for that connector in that month.
+- mode: NULLABLE
+  name: free_active_rows
+  type: INTEGER
+  description: |
+    Number of free active rows for that connector in that month.
+- mode: NULLABLE
+  name: total_active_rows
+  type: INTEGER
+  description: |
+    Total number of active rows, billed and free for that connector in that month.
+- mode: NULLABLE
+  name: percentage_of_destination_paid_active_rows
+  type: FLOAT
+  description: |
+    Percentage of paid active rows for that connector compared to the total amount of paid active rows for the destination for that month.
+- mode: NULLABLE
+  name: percentage_of_total_active_rows
+  type: FLOAT
+  description: |
+    Percentage of paid active rows for that connector compared to the total amount of paid active rows over all destinations for that month.
+- mode: NULLABLE
+  name: cost_in_usd
+  type: FLOAT
+  description: |
+    Costs for that connector for that month in USD
+- mode: NULLABLE
+  name: cost_in_credits
+  type: FLOAT
+  description: |
+    Costs for that connector for that month in Fivetran credits

sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_costs_v1/metadata.yaml

@@ -0,0 +1,14 @@
+friendly_name: Monthly Costs
+description: |-
+  Monthly Cost for fivetran connectors. Before September 2022 the costs are given in credits, after in USD. populated from Fivetran Logs
+owners:
+- lschiestl@mozilla.com
+labels: {}
+scheduling:
+  dag_name: bqetl_fivetran_costs
+  date_partition_parameter: null
+  depends_on_fivetran:
+  - task_id: fivetran_log_prod
+  - task_id: fivetran_log_dev
+bigquery: null
+references: {}

sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_costs_v1/query.sql

@@ -0,0 +1,53 @@
+WITH monthly_costs_dollars AS (
+  SELECT
+    destination_id,
+    CAST(CONCAT(measured_month, "-01") AS DATE) AS measured_month,
+    amount AS dollars_spent
+  FROM
+    `moz-fx-data-bq-fivetran.fivetran_log.usage_cost`
+  UNION ALL
+  SELECT
+    destination_id,
+    CAST(CONCAT(measured_month, "-01") AS DATE) AS measured_month,
+    amount AS dollars_spent
+  FROM
+    `dev-fivetran.fivetran_log.usage_cost`
+),
+monthly_costs_credits AS (
+  SELECT
+    destination_id,
+    CAST(CONCAT(measured_month, "-01") AS DATE) AS measured_month,
+    credits_consumed AS credits_spent
+  FROM
+    `moz-fx-data-bq-fivetran.fivetran_log.credits_used`
+  UNION ALL
+  SELECT
+    destination_id,
+    CAST(CONCAT(measured_month, "-01") AS DATE) AS measured_month,
+    credits_consumed AS credits_spent
+  FROM
+    `dev-fivetran.fivetran_log.credits_used`
+),
+monthly_costs AS (
+  SELECT
+    COALESCE(
+      monthly_costs_credits.destination_id,
+      monthly_costs_dollars.destination_id
+    ) AS destination_id,
+    COALESCE(
+      monthly_costs_credits.measured_month,
+      monthly_costs_dollars.measured_month
+    ) AS measured_month,
+    monthly_costs_credits.credits_spent,
+    monthly_costs_dollars.dollars_spent
+  FROM
+    monthly_costs_credits
+  FULL OUTER JOIN
+    monthly_costs_dollars
+  USING
+    (destination_id, measured_month)
+)
+SELECT
+  *
+FROM
+  monthly_costs

sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_costs_v1/schema.yaml

@@ -0,0 +1,21 @@
+fields:
+- mode: NULLABLE
+  name: destination_id
+  type: STRING
+  description: |
+    Foreign key referencing the destination warehouse that was used.
+- mode: NULLABLE
+  name: measured_month
+  type: DATE
+  description: |
+    Month of the usage activity.
+- mode: NULLABLE
+  name: credits_spent
+  type: INTEGER
+  description: |
+    Total credits used by the destination for the given month. Only used until September 2022 when Fivetran switched to USD.
+- mode: NULLABLE
+  name: dollars_spent
+  type: NUMERIC
+  description: |
+    Total USD used by the destination for the given month. Since September 2022 Fivetran gives the costs in USD.

tests/sql/moz-fx-data-shared-prod/fivetran_costs_derived/destinations_v1/test_union/dev-fivetran.fivetran_log.destination.yaml

@@ -0,0 +1,6 @@
+---
+- id: "test_dev_destination_id"
+  name: "test_dev_destination_name"
+  account_id: "test_account_id"
+  created_at: "2023-01-01"
+  region: "us-east4"

tests/sql/moz-fx-data-shared-prod/fivetran_costs_derived/destinations_v1/test_union/expect.yaml

@@ -0,0 +1,10 @@
+---
+- &base
+  destination_id: "test_prod_destination_id"
+  destination_name: "test_prod_destination_name"
+  account_id: "test_account_id"
+  created_at: "2023-01-01"
+  region: "us-east4"
+- <<: *base
+  destination_id: "test_dev_destination_id"
+  destination_name: "test_dev_destination_name"

tests/sql/moz-fx-data-shared-prod/fivetran_costs_derived/destinations_v1/test_union/moz-fx-data-bq-fivetran.fivetran_log.destination.yaml

@@ -0,0 +1,6 @@
+---
+- id: "test_prod_destination_id"
+  name: "test_prod_destination_name"
+  account_id: "test_account_id"
+  created_at: "2023-01-01"
+  region: "us-east4"

tests/sql/moz-fx-data-shared-prod/fivetran_costs_derived/incremental_mar_v1/test_union/dev-fivetran.fivetran_log.incremental_mar.yaml

@@ -0,0 +1,14 @@
+---
+- &base
+  measured_date: "2023-01-03"
+  destination_id: "test_dev_destination_id"
+  connector_id: "test_connector_id_2"
+  table_name: "table_2"
+  free_type: "PAID"
+  incremental_rows: 31
+- <<: *base
+  measured_date: "2023-01-04"
+  incremental_rows: 51
+- <<: *base
+  measured_date: "2023-01-05"
+  incremental_rows: 10

tests/sql/moz-fx-data-shared-prod/fivetran_costs_derived/incremental_mar_v1/test_union/expect.yaml

@@ -0,0 +1,44 @@
+---
+- &dev_base
+  measured_date: "2023-01-03"
+  measured_month: "2023-01-01"
+  destination_id: "test_dev_destination_id"
+  connector_name: "test_connector_id_2"
+  table_name: "table_2"
+  paid_active_rows: 31
+  free_active_rows: 0
+  total_active_rows: 31
+- <<: *dev_base
+  measured_date: "2023-01-04"
+  paid_active_rows: 51
+  free_active_rows: 0
+  total_active_rows: 51
+- <<: *dev_base
+  measured_date: "2023-01-05"
+  paid_active_rows: 10
+  free_active_rows: 0
+  total_active_rows: 10
+- &prod_base
+  measured_date: "2023-01-03"
+  measured_month: "2023-01-01"
+  destination_id: "test_prod_destination_id"
+  connector_name: "test_connector_id_1"
+  table_name: "table_1"
+  paid_active_rows: 3
+  free_active_rows: 0
+  total_active_rows: 3
+- <<: *prod_base
+  measured_date: "2023-01-04"
+  paid_active_rows: 5
+  free_active_rows: 0
+  total_active_rows: 5
+- <<: *prod_base
+  measured_date: "2023-01-05"
+  paid_active_rows: 100
+  free_active_rows: 0
+  total_active_rows: 100
+- <<: *prod_base
+  measured_date: "2023-01-06"
+  paid_active_rows: 0
+  free_active_rows: 10
+  total_active_rows: 10

tests/sql/moz-fx-data-shared-prod/fivetran_costs_derived/incremental_mar_v1/test_union/moz-fx-data-bq-fivetran.fivetran_log.incremental_mar.yaml

@@ -0,0 +1,18 @@
+---
+- &base
+  measured_date: "2023-01-03"
+  destination_id: "test_prod_destination_id"
+  connector_id: "test_connector_id_1"
+  table_name: "table_1"
+  free_type: "PAID"
+  incremental_rows: 3
+- <<: *base
+  measured_date: "2023-01-04"
+  incremental_rows: 5
+- <<: *base
+  measured_date: "2023-01-05"
+  incremental_rows: 100
+- <<: *base
+  measured_date: "2023-01-06"
+  free_type: "FREE"
+  incremental_rows: 10

tests/sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_connector_costs_v1/moz-fx-data-shared-prod.fivetran_costs_derived.destinations_v1.schema.json

@@ -0,0 +1,12 @@
+[
+    {
+        "type": "STRING",
+        "name": "destination_id",
+        "mode": "NULLABLE"
+    },
+    {
+        "type": "STRING",
+        "name": "destination_name",
+        "mode": "NULLABLE"
+    }
+]

tests/sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_connector_costs_v1/moz-fx-data-shared-prod.fivetran_costs_derived.incremental_mar_v1.schema.json

@@ -0,0 +1,32 @@
+[
+    {
+        "type": "STRING",
+        "name": "destination_id",
+        "mode": "NULLABLE"
+    },
+    {
+        "type": "DATE",
+        "name": "measured_month",
+        "mode": "NULLABLE"
+    },
+    {
+        "type": "STRING",
+        "name": "connector_name",
+        "mode": "NULLABLE"
+    },
+    {
+        "type": "INTEGER",
+        "name": "paid_active_rows",
+        "mode": "NULLABLE"
+    },
+    {
+        "type": "INTEGER",
+        "name": "free_active_rows",
+        "mode": "NULLABLE"
+    },
+    {
+        "type": "INTEGER",
+        "name": "total_active_rows",
+        "mode": "NULLABLE"
+    }
+]

tests/sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_connector_costs_v1/moz-fx-data-shared-prod.fivetran_costs_derived.monthly_costs_v1.schema.json

@@ -0,0 +1,23 @@
+[
+    {
+        "type": "STRING",
+        "name": "destination_id",
+        "mode": "NULLABLE"
+    },
+    {
+        "type": "DATE",
+        "name": "measured_month",
+        "mode": "NULLABLE"
+    },
+    {
+        "type": "INTEGER",
+        "name": "credits_spent",
+        "mode": "NULLABLE"
+    },
+    {
+        "type": "NUMERIC",
+        "name": "dollars_spent",
+        "mode": "NULLABLE"
+    }
+
+]

tests/sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_connector_costs_v1/test_one_connector_mutliple_months/expect.yaml

@@ -0,0 +1,69 @@
+---
+- &credits_base
+  destination_name: dev_destination_name
+  measured_month: 2022-01-01
+  connector_name: connector_1
+  paid_active_rows: 10
+  free_active_rows: 0
+  total_active_rows: 10
+  percentage_of_total_paid_mar: 100.0
+  cost_in_credit: 10
+- <<: *credits_base
+  measured_month: 2022-02-01
+  paid_active_rows: 20
+  total_active_rows: 20
+  cost_in_credit: 20
+- <<: *credits_base
+  measured_month: 2022-03-01
+  paid_active_rows: 30
+  total_active_rows: 30
+  cost_in_credit: 30
+- <<: *credits_base
+  measured_month: 2022-04-01
+  paid_active_rows: 40
+  total_active_rows: 40
+  cost_in_credit: 40
+- <<: *credits_base
+  measured_month: 2022-05-01
+  paid_active_rows: 50
+  total_active_rows: 50
+  cost_in_credit: 50
+- <<: *credits_base
+  measured_month: 2022-06-01
+  paid_active_rows: 60
+  total_active_rows: 60
+  cost_in_credit: 60
+- <<: *credits_base
+  measured_month: 2022-07-01
+  paid_active_rows: 70
+  total_active_rows: 70
+  cost_in_credit: 70
+- <<: *credits_base
+  measured_month: 2022-08-01
+  paid_active_rows: 80
+  total_active_rows: 80
+  cost_in_credit: 80
+- &dollars_base
+  destination_name: dev_destination_name
+  measured_month: 2022-09-01
+  connector_name: connector_1
+  paid_active_rows: 90
+  free_active_rows: 0
+  total_active_rows: 90
+  percentage_of_total_paid_mar: 100.0
+  cost_in_usd: 900
+- <<: *dollars_base
+  measured_month: 2022-10-01
+  paid_active_rows: 100
+  total_active_rows: 100
+  cost_in_usd: 1000
+- <<: *dollars_base
+  measured_month: 2022-11-01
+  paid_active_rows: 110
+  total_active_rows: 110
+  cost_in_usd: 1100
+- <<: *dollars_base
+  measured_month: 2022-12-01
+  paid_active_rows: 120
+  total_active_rows: 120
+  cost_in_usd: 1200

tests/sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_connector_costs_v1/test_one_connector_mutliple_months/moz-fx-data-shared-prod.fivetran_costs_derived.destinations_v1.yaml

@@ -0,0 +1,3 @@
+---
+- destination_id: dev_destination_id
+  destination_name: dev_destination_name

tests/sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_connector_costs_v1/test_one_connector_mutliple_months/moz-fx-data-shared-prod.fivetran_costs_derived.incremental_mar_v1.yaml

@@ -0,0 +1,52 @@
+---
+- &base
+  destination_id: dev_destination_id
+  measured_month: 2022-01-01
+  connector_name: connector_1
+  paid_active_rows: 10
+  free_active_rows: 0
+  total_active_rows: 10
+- <<: *base
+  measured_month: 2022-02-01
+  paid_active_rows: 20
+  total_active_rows: 20
+- <<: *base
+  measured_month: 2022-03-01
+  paid_active_rows: 30
+  total_active_rows: 30
+- <<: *base
+  measured_month: 2022-04-01
+  paid_active_rows: 40
+  total_active_rows: 40
+- <<: *base
+  measured_month: 2022-05-01
+  paid_active_rows: 50
+  total_active_rows: 50
+- <<: *base
+  measured_month: 2022-06-01
+  paid_active_rows: 60
+  total_active_rows: 60
+- <<: *base
+  measured_month: 2022-07-01
+  paid_active_rows: 70
+  total_active_rows: 70
+- <<: *base
+  measured_month: 2022-08-01
+  paid_active_rows: 80
+  total_active_rows: 80
+- <<: *base
+  measured_month: 2022-09-01
+  paid_active_rows: 90
+  total_active_rows: 90
+- <<: *base
+  measured_month: 2022-10-01
+  paid_active_rows: 100
+  total_active_rows: 100
+- <<: *base
+  measured_month: 2022-11-01
+  paid_active_rows: 110
+  total_active_rows: 110
+- <<: *base
+  measured_month: 2022-12-01
+  paid_active_rows: 120
+  total_active_rows: 120

tests/sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_connector_costs_v1/test_one_connector_mutliple_months/moz-fx-data-shared-prod.fivetran_costs_derived.monthly_costs_v1.yaml

@@ -0,0 +1,41 @@
+---
+- &credits_base
+  destination_id: dev_destination_id
+  measured_month: 2022-01-01
+  credits_spent: 10
+  dollars_spent: null
+- <<: *credits_base
+  measured_month: 2022-02-01
+  credits_spent: 20
+- <<: *credits_base
+  measured_month: 2022-03-01
+  credits_spent: 30
+- <<: *credits_base
+  measured_month: 2022-04-01
+  credits_spent: 40
+- <<: *credits_base
+  measured_month: 2022-05-01
+  credits_spent: 50
+- <<: *credits_base
+  measured_month: 2022-06-01
+  credits_spent: 60
+- <<: *credits_base
+  measured_month: 2022-07-01
+  credits_spent: 70
+- <<: *credits_base
+  measured_month: 2022-08-01
+  credits_spent: 80
+- &dollars_base
+  <<: *credits_base
+  measured_month: 2022-09-01
+  credits_spent: null
+  dollars_spent: 900
+- <<: *dollars_base
+  measured_month: 2022-10-01
+  dollars_spent: 1000
+- <<: *dollars_base
+  measured_month: 2022-11-01
+  dollars_spent: 1100
+- <<: *dollars_base
+  measured_month: 2022-12-01
+  dollars_spent: 1200

tests/sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_connector_costs_v1/test_one_connector_no_matching_destination/expect.yaml

@@ -0,0 +1,8 @@
+---
+- measured_month: 2023-01-01
+  connector_name: connector_1
+  paid_active_rows: 10
+  free_active_rows: 0
+  total_active_rows: 10
+  percentage_of_total_paid_mar: 100.0
+  cost_in_usd: 100.0

tests/sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_connector_costs_v1/test_one_connector_no_matching_destination/moz-fx-data-shared-prod.fivetran_costs_derived.destinations_v1.yaml

@@ -0,0 +1,3 @@
+---
+- destination_id: other_destination_id
+  destination_name: other_destination_name

tests/sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_connector_costs_v1/test_one_connector_no_matching_destination/moz-fx-data-shared-prod.fivetran_costs_derived.incremental_mar_v1.yaml

@@ -0,0 +1,7 @@
+---
+- destination_id: dev_destination_id
+  measured_month: 2023-01-01
+  connector_name: connector_1
+  paid_active_rows: 10
+  free_active_rows: 0
+  total_active_rows: 10

tests/sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_connector_costs_v1/test_one_connector_no_matching_destination/moz-fx-data-shared-prod.fivetran_costs_derived.monthly_costs_v1.yaml

@@ -0,0 +1,5 @@
+---
+- destination_id: dev_destination_id
+  measured_month: 2023-01-01
+  credits_spent: null
+  dollars_spent: 100

tests/sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_connector_costs_v1/test_three_connectors_one_month/expect.yaml

@@ -0,0 +1,24 @@
+---
+- &base
+  destination_name: dev_destination_name
+  measured_month: 2023-01-01
+  connector_name: connector_1
+  paid_active_rows: 30
+  free_active_rows: 10
+  total_active_rows: 40
+  percentage_of_total_paid_mar: 30.0
+  cost_in_usd: 30.0
+- <<: *base
+  connector_name: connector_2
+  paid_active_rows: 70
+  free_active_rows: 0
+  total_active_rows: 70
+  percentage_of_total_paid_mar: 70.0
+  cost_in_usd: 70.0
+- <<: *base
+  connector_name: connector_3
+  paid_active_rows: 0
+  free_active_rows: 40
+  total_active_rows: 40
+  percentage_of_total_paid_mar: 0
+  cost_in_usd: 0

tests/sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_connector_costs_v1/test_three_connectors_one_month/moz-fx-data-shared-prod.fivetran_costs_derived.destinations_v1.yaml

@@ -0,0 +1,3 @@
+---
+- destination_id: dev_destination_id
+  destination_name: dev_destination_name

tests/sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_connector_costs_v1/test_three_connectors_one_month/moz-fx-data-shared-prod.fivetran_costs_derived.incremental_mar_v1.yaml

@@ -0,0 +1,35 @@
+---
+- &con_1_base
+  destination_id: dev_destination_id
+  measured_month: 2023-01-01
+  connector_name: connector_1
+  paid_active_rows: 10
+  free_active_rows: 0
+  total_active_rows: 10
+- <<: *con_1_base
+  paid_active_rows: 20
+  total_active_rows: 20
+- <<: *con_1_base
+  paid_active_rows: 0
+  free_active_rows: 10
+  total_active_rows: 10
+- &con_2_base
+  destination_id: dev_destination_id
+  measured_month: 2023-01-01
+  connector_name: connector_2
+  paid_active_rows: 50
+  free_active_rows: 0
+  total_active_rows: 50
+- <<: *con_2_base
+  paid_active_rows: 20
+  total_active_rows: 20
+- &con_3_base
+  destination_id: dev_destination_id
+  measured_month: 2023-01-01
+  connector_name: connector_3
+  paid_active_rows: 0
+  free_active_rows: 10
+  total_active_rows: 10
+- <<: *con_3_base
+  free_active_rows: 30
+  total_active_rows: 30

tests/sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_connector_costs_v1/test_three_connectors_one_month/moz-fx-data-shared-prod.fivetran_costs_derived.monthly_costs_v1.yaml

@@ -0,0 +1,5 @@
+---
+- destination_id: dev_destination_id
+  measured_month: 2023-01-01
+  credits_spent: null
+  dollars_spent: 100

tests/sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_costs_v1/test_union/dev-fivetran.fivetran_log.credits_used.yaml

@@ -0,0 +1,4 @@
+---
+- destination_id: dev_destination_id
+  measured_month: 2023-01
+  credits_consumed: 32

tests/sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_costs_v1/test_union/dev-fivetran.fivetran_log.usage_cost.yaml

@@ -0,0 +1,7 @@
+---
+- destination_id: dev_destination_id
+  measured_month: 2023-01
+  amount: 10.23
+- destination_id: dev_destination_id
+  measured_month: 2022-12
+  amount: 5.0

tests/sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_costs_v1/test_union/expect.yaml

@@ -0,0 +1,15 @@
+---
+- destination_id: prod_destination_id
+  measured_month: "2023-01-01"
+  credits_spent: 3
+  dollars_spent: 14.40
+- destination_id: prod_destination_id
+  measured_month: 2022-02-01
+  credits_spent: 5
+- destination_id: dev_destination_id
+  measured_month: 2023-01-01
+  credits_spent: 32
+  dollars_spent: 10.23
+- destination_id: dev_destination_id
+  measured_month: 2022-12-01
+  dollars_spent: 5.0

tests/sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_costs_v1/test_union/moz-fx-data-bq-fivetran.fivetran_log.credits_used.yaml

@@ -0,0 +1,7 @@
+---
+- destination_id: prod_destination_id
+  measured_month: 2023-01
+  credits_consumed: 3
+- destination_id: prod_destination_id
+  measured_month: 2022-02
+  credits_consumed: 5

tests/sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_costs_v1/test_union/moz-fx-data-bq-fivetran.fivetran_log.usage_cost.yaml

@@ -0,0 +1,4 @@
+---
+- destination_id: prod_destination_id
+  measured_month: 2023-01
+  amount: 14.40