Use the shared-prod URL for the dryrun script

This commit is contained in:
Jeff Klukas 2019-10-08 15:53:06 -04:00
Родитель e872a76860
Коммит 2e821fbdc1
3 изменённых файлов: 31 добавлений и 8 удалений

Просмотреть файл

@ -20,6 +20,9 @@ import sys
DRY_RUN_URL = (
"https://us-central1-moz-fx-data-shared-prod.cloudfunctions.net/bigquery-etl-dryrun"
)
DERIVED_DATASETS_DRY_RUN_URL = (
"https://us-central1-moz-fx-data-derived-datasets.cloudfunctions.net/"
"bigquery-etl-dryrun"
)
@ -49,13 +52,33 @@ SKIP = {
"sql/telemetry_derived/addons_aggregates_v3/query.sql",
}
USE_DERIVED_DATASETS = {
"sql/telemetry_derived/ssl_ratios_v1/query.sql",
"sql/telemetry_derived/clients_last_seen_v1/query.sql",
"sql/telemetry_derived/clients_daily_v7/query.sql",
"sql/telemetry/fxa_users_last_seen_raw_v1/query.sql",
"sql/telemetry/clients_daily_keyed_boolean_aggregates_v1/query.sql",
"sql/telemetry/clients_daily_scalar_aggregates_v1/query.sql",
"sql/telemetry/client_histogram_probe_counts_v1/query.sql",
"sql/telemetry/clients_daily_keyed_scalar_aggregates_v1/query.sql",
"sql/telemetry/firefox_accounts_exact_mau28_raw_v1/query.sql",
"sql/telemetry/core_clients_daily_v1/query.sql",
"sql/telemetry/client_scalar_probe_counts_v1/query.sql",
"sql/telemetry/firefox_kpi_dashboard_v1/query.sql",
"sql/telemetry/core_clients_last_seen_raw_v1/query.sql",
}
def worker_entrypoint(sqlfile):
sql = open(sqlfile).read()
if sqlfile in USE_DERIVED_DATASETS:
url = DERIVED_DATASETS_DRY_RUN_URL
else:
url = DRY_RUN_URL
try:
r = urlopen(
Request(
DRY_RUN_URL,
url,
headers={"Content-Type": "application/json"},
data=json.dumps(
{"dataset": basename(dirname(dirname(sqlfile))), "query": sql}
@ -82,7 +105,7 @@ def worker_entrypoint(sqlfile):
# We want the dryrun service to only have read permissions, so
# we expect CREATE VIEW and CREATE TABLE to throw specific
# exceptions.
print(f"{sqlfile:59} OK, but with insufficient access to create table/view")
print(f"{sqlfile:59} OK")
else:
print(f"{sqlfile:59} ERROR\n", response["errors"])
return False

Просмотреть файл

@ -1,10 +1,10 @@
WITH
base AS (
SELECT * FROM `moz-fx-data-shared-prod.telemetry_derived.smoot_usage_desktop_v2`
SELECT * FROM smoot_usage_desktop_v2
UNION ALL
SELECT * FROM `moz-fx-data-shared-prod.telemetry_derived.smoot_usage_nondesktop_v2`
SELECT * FROM smoot_usage_nondesktop_v2
UNION ALL
SELECT * FROM `moz-fx-data-shared-prod.telemetry_derived.smoot_usage_fxa_v2`
SELECT * FROM smoot_usage_fxa_v2
)
--
SELECT

Просмотреть файл

@ -1,10 +1,10 @@
WITH
base AS (
SELECT * FROM `moz-fx-data-shared-prod.telemetry_derived.smoot_usage_desktop_v2`
SELECT * FROM smoot_usage_desktop_v2
UNION ALL
SELECT * FROM `moz-fx-data-shared-prod.telemetry_derived.smoot_usage_nondesktop_v2`
SELECT * FROM smoot_usage_nondesktop_v2
UNION ALL
SELECT * FROM `moz-fx-data-shared-prod.telemetry_derived.smoot_usage_fxa_v2`
SELECT * FROM smoot_usage_fxa_v2
)
--
SELECT