Родитель
505c895f62
Коммит
1d3030e698
|
@ -21,8 +21,8 @@ RUN pip install --no-deps -r requirements.txt
|
|||
FROM google/cloud-sdk:${GOOGLE_CLOUD_SDK_VERSION}-alpine AS google-cloud-sdk
|
||||
|
||||
FROM base
|
||||
# add bash for entrypoint and jdk for jni access to zetasql
|
||||
RUN mkdir -p /usr/share/man/man1 && apt-get update -qqy && apt-get install -qqy bash default-jdk-headless git
|
||||
# add bash for entrypoint
|
||||
RUN mkdir -p /usr/share/man/man1 && apt-get update -qqy && apt-get install -qqy bash git
|
||||
COPY --from=google-cloud-sdk /google-cloud-sdk /google-cloud-sdk
|
||||
ENV PATH /google-cloud-sdk/bin:$PATH
|
||||
COPY --from=python-deps /usr/local /usr/local
|
||||
|
|
|
@ -438,7 +438,7 @@ class Task:
|
|||
)
|
||||
|
||||
def _get_referenced_tables(self):
|
||||
"""Use zetasql to get tables the query depends on."""
|
||||
"""Use sqlglot to get tables the query depends on."""
|
||||
logging.info(f"Get dependencies for {self.task_key}")
|
||||
|
||||
if self.is_python_script:
|
||||
|
|
|
@ -74,9 +74,6 @@ initial_invoices AS (
|
|||
`moz-fx-data-shared-prod`.stripe_external.nonprod_invoice_v1 AS invoices
|
||||
USING
|
||||
(subscription_id)
|
||||
-- ZetaSQL requires QUALIFY to be used in conjunction with WHERE, GROUP BY, or HAVING.
|
||||
WHERE
|
||||
TRUE
|
||||
QUALIFY
|
||||
1 = ROW_NUMBER() OVER (PARTITION BY subscription_id ORDER BY invoices.created)
|
||||
),
|
||||
|
|
|
@ -59,9 +59,6 @@ initial_invoices AS (
|
|||
`moz-fx-data-shared-prod`.stripe_external.invoice_v1 AS invoices
|
||||
USING
|
||||
(subscription_id)
|
||||
-- ZetaSQL requires QUALIFY to be used in conjunction with WHERE, GROUP BY, or HAVING.
|
||||
WHERE
|
||||
TRUE
|
||||
QUALIFY
|
||||
1 = ROW_NUMBER() OVER (PARTITION BY subscription_id ORDER BY invoices.created)
|
||||
),
|
||||
|
|
|
@ -49,8 +49,6 @@ new_events AS (
|
|||
"New" AS event_type,
|
||||
FROM
|
||||
`moz-fx-data-shared-prod`.hubs.active_subscription_ids
|
||||
WHERE
|
||||
TRUE -- zetasql requires QUALIFY to be used in conjunction with WHERE, GROUP BY, or HAVING
|
||||
QUALIFY
|
||||
LAG(active_date) OVER (PARTITION BY subscription_id ORDER BY active_date) IS DISTINCT FROM (
|
||||
active_date - 1
|
||||
|
@ -65,8 +63,6 @@ cancelled_events AS (
|
|||
`moz-fx-data-shared-prod`.hubs.active_subscription_ids
|
||||
CROSS JOIN
|
||||
max_active_date
|
||||
WHERE
|
||||
TRUE -- zetasql requires QUALIFY to be used in conjunction with WHERE, GROUP BY, or HAVING
|
||||
QUALIFY
|
||||
LEAD(active_date) OVER (PARTITION BY subscription_id ORDER BY active_date) IS DISTINCT FROM (
|
||||
active_date + 1
|
||||
|
|
|
@ -49,8 +49,6 @@ new_events AS (
|
|||
"New" AS event_type,
|
||||
FROM
|
||||
`moz-fx-data-shared-prod`.mozilla_vpn.active_subscription_ids
|
||||
WHERE
|
||||
TRUE -- zetasql requires QUALIFY to be used in conjunction with WHERE, GROUP BY, or HAVING
|
||||
QUALIFY
|
||||
LAG(active_date) OVER (PARTITION BY subscription_id ORDER BY active_date) IS DISTINCT FROM (
|
||||
active_date - 1
|
||||
|
@ -65,8 +63,6 @@ cancelled_events AS (
|
|||
`moz-fx-data-shared-prod`.mozilla_vpn.active_subscription_ids
|
||||
CROSS JOIN
|
||||
max_active_date
|
||||
WHERE
|
||||
TRUE -- zetasql requires QUALIFY to be used in conjunction with WHERE, GROUP BY, or HAVING
|
||||
QUALIFY
|
||||
LEAD(active_date) OVER (PARTITION BY subscription_id ORDER BY active_date) IS DISTINCT FROM (
|
||||
active_date + 1
|
||||
|
|
|
@ -49,8 +49,6 @@ new_events AS (
|
|||
"New" AS event_type,
|
||||
FROM
|
||||
`moz-fx-data-shared-prod`.relay.active_subscription_ids
|
||||
WHERE
|
||||
TRUE -- zetasql requires QUALIFY to be used in conjunction with WHERE, GROUP BY, or HAVING
|
||||
QUALIFY
|
||||
LAG(active_date) OVER (PARTITION BY subscription_id ORDER BY active_date) IS DISTINCT FROM (
|
||||
active_date - 1
|
||||
|
@ -65,8 +63,6 @@ cancelled_events AS (
|
|||
`moz-fx-data-shared-prod`.relay.active_subscription_ids
|
||||
CROSS JOIN
|
||||
max_active_date
|
||||
WHERE
|
||||
TRUE -- zetasql requires QUALIFY to be used in conjunction with WHERE, GROUP BY, or HAVING
|
||||
QUALIFY
|
||||
LEAD(active_date) OVER (PARTITION BY subscription_id ORDER BY active_date) IS DISTINCT FROM (
|
||||
active_date + 1
|
||||
|
|
|
@ -119,9 +119,6 @@ subscription_items AS (
|
|||
plan_id,
|
||||
FROM
|
||||
`moz-fx-data-shared-prod`.stripe_external.nonprod_subscription_item_v1
|
||||
-- ZetaSQL requires QUALIFY to be used in conjunction with WHERE, GROUP BY, or HAVING.
|
||||
WHERE
|
||||
TRUE
|
||||
QUALIFY
|
||||
-- With how our subscription platform currently works each Stripe subscription should
|
||||
-- only have one subscription item, and we enforce that so the ETL can rely on it.
|
||||
|
|
|
@ -119,9 +119,6 @@ subscription_items AS (
|
|||
plan_id,
|
||||
FROM
|
||||
`moz-fx-data-shared-prod`.stripe_external.subscription_item_v1
|
||||
-- ZetaSQL requires QUALIFY to be used in conjunction with WHERE, GROUP BY, or HAVING.
|
||||
WHERE
|
||||
TRUE
|
||||
QUALIFY
|
||||
-- With how our subscription platform currently works each Stripe subscription should
|
||||
-- only have one subscription item, and we enforce that so the ETL can rely on it.
|
||||
|
|
|
@ -92,8 +92,6 @@ metrics_windowed AS (
|
|||
FROM
|
||||
metrics_unioned,
|
||||
UNNEST(metrics)
|
||||
WHERE
|
||||
TRUE
|
||||
QUALIFY
|
||||
ROW_NUMBER() OVER (PARTITION BY channel, metric_name) = 1
|
||||
),
|
||||
|
@ -187,8 +185,6 @@ baseline_windowed AS (
|
|||
FROM
|
||||
baseline_unioned,
|
||||
UNNEST(metrics)
|
||||
WHERE
|
||||
TRUE
|
||||
QUALIFY
|
||||
ROW_NUMBER() OVER (PARTITION BY channel, metric_name) = 1
|
||||
),
|
||||
|
@ -285,8 +281,6 @@ windowed AS (
|
|||
FROM
|
||||
unioned,
|
||||
UNNEST(metrics)
|
||||
WHERE
|
||||
TRUE
|
||||
QUALIFY
|
||||
ROW_NUMBER() OVER (PARTITION BY channel, metric_name) = 1
|
||||
)
|
||||
|
|
|
@ -100,8 +100,6 @@ SELECT
|
|||
sample_id,
|
||||
client_id
|
||||
FROM _joined
|
||||
WHERE
|
||||
TRUE
|
||||
QUALIFY
|
||||
IF(
|
||||
COUNT(*) OVER (PARTITION BY client_id) > 1,
|
||||
|
|
Загрузка…
Ссылка в новой задаче