Add ConfigLoader and move dry run skip to bqetl_project.yaml (#4000)
* Add ConfigLoader and move dry run skip to bqetl_project.yaml * format tests
This commit is contained in:
Anna Scholtz
GitHub
Родитель
6f1066e0bc
Коммит
d9bda0df7e
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
import logging
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
|
||||
|
|
@ -19,6 +20,7 @@ from ..cli.query import query
|
|||
from ..cli.routine import mozfun, routine
|
||||
from ..cli.stage import stage
|
||||
from ..cli.view import view
|
||||
from ..config import ConfigLoader
|
||||
from ..copy_deduplicate import copy_deduplicate
|
||||
from ..dependency import dependency
|
||||
from ..docs import docs_
|
||||
|
|
@ -73,4 +75,5 @@ def cli(prog_name=None):
|
|||
|
||||
|
||||
if __name__ == "__main__":
|
||||
ConfigLoader.set_project_dir(Path().absolute())
|
||||
cli()
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import click
|
|||
from google.cloud import bigquery
|
||||
|
||||
from ..cli.utils import is_authenticated
|
||||
from ..dryrun import SKIP, DryRun
|
||||
from ..dryrun import DryRun
|
||||
|
||||
|
||||
@click.command(
|
||||
|
|
@ -55,7 +55,7 @@ from ..dryrun import SKIP, DryRun
|
|||
)
|
||||
@click.option(
|
||||
"--respect-skip/--ignore-skip",
|
||||
help="Respect or ignore query SKIP configuration. Default is --respect-skip.",
|
||||
help="Respect or ignore query skip configuration. Default is --respect-skip.",
|
||||
default=True,
|
||||
)
|
||||
@click.option(
|
||||
|
|
@ -71,7 +71,7 @@ def dryrun(
|
|||
project: str,
|
||||
):
|
||||
"""Perform a dry run."""
|
||||
file_names = ("query.sql", "view.sql", "part*.sql", "init.sql")
|
||||
file_names = ("query.sql", "view.sql", "part*.sql", "init.sql", "checks.sql")
|
||||
file_re = re.compile("|".join(map(fnmatch.translate, file_names)))
|
||||
|
||||
sql_files: Set[str] = set()
|
||||
|
|
@ -89,7 +89,7 @@ def dryrun(
|
|||
click.echo(f"Invalid path {path}", err=True)
|
||||
sys.exit(1)
|
||||
if respect_skip:
|
||||
sql_files -= SKIP
|
||||
sql_files -= DryRun.skipped_files()
|
||||
|
||||
if not sql_files:
|
||||
print("Skipping dry run because no queries matched")
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ from ..cli.utils import (
|
|||
use_cloud_function_option,
|
||||
)
|
||||
from ..dependency import get_dependency_graph
|
||||
from ..dryrun import SKIP, DryRun
|
||||
from ..dryrun import DryRun
|
||||
from ..format_sql.format import SKIP as SKIP_FORMAT
|
||||
from ..format_sql.formatter import reformat
|
||||
from ..metadata import validate_metadata
|
||||
|
|
@ -1569,7 +1569,7 @@ def _update_query_schema(
|
|||
|
||||
Return True if the schema changed, False if it is unchanged.
|
||||
"""
|
||||
if respect_dryrun_skip and str(query_file) in SKIP:
|
||||
if respect_dryrun_skip and str(query_file) in DryRun.skipped_files():
|
||||
click.echo(f"{query_file} dry runs are skipped. Cannot update schemas.")
|
||||
return
|
||||
|
||||
|
|
@ -1838,7 +1838,7 @@ def deploy(
|
|||
)
|
||||
|
||||
def _deploy(query_file):
|
||||
if respect_dryrun_skip and str(query_file) in SKIP:
|
||||
if respect_dryrun_skip and str(query_file) in DryRun.skipped_files():
|
||||
click.echo(f"{query_file} dry runs are skipped. Cannot validate schemas.")
|
||||
return
|
||||
|
||||
|
|
|
|||
|
|
@ -10,12 +10,12 @@ from pathlib import Path
|
|||
import click
|
||||
from google.cloud import bigquery
|
||||
|
||||
from .. import dryrun
|
||||
from ..cli.query import deploy as deploy_query_schema
|
||||
from ..cli.query import update as update_query_schema
|
||||
from ..cli.routine import publish as publish_routine
|
||||
from ..cli.utils import paths_matching_name_pattern, sql_dir_option
|
||||
from ..cli.view import publish as publish_view
|
||||
from ..dryrun import DryRun
|
||||
from ..routine.parse_routine import (
|
||||
UDF_FILE,
|
||||
RawRoutine,
|
||||
|
|
@ -186,9 +186,6 @@ def deploy(
|
|||
if artifact_file.parent.exists():
|
||||
shutil.rmtree(artifact_file.parent)
|
||||
|
||||
# update dryrun skip list
|
||||
dryrun.add_test_project_to_skip(sql_dir, project_id)
|
||||
|
||||
# deploy to stage
|
||||
_deploy_artifacts(ctx, updated_artifact_files, project_id, dataset_suffix, sql_dir)
|
||||
|
||||
|
|
@ -368,7 +365,7 @@ def _deploy_artifacts(ctx, artifact_files, project_id, dataset_suffix, sql_dir):
|
|||
name=str(query_file),
|
||||
sql_dir=sql_dir,
|
||||
project_id=project_id,
|
||||
respect_dryrun_skip=False,
|
||||
respect_dryrun_skip=True,
|
||||
)
|
||||
ctx.invoke(
|
||||
deploy_query_schema,
|
||||
|
|
@ -387,7 +384,7 @@ def _deploy_artifacts(ctx, artifact_files, project_id, dataset_suffix, sql_dir):
|
|||
view_files = [
|
||||
file
|
||||
for file in artifact_files
|
||||
if file.name == VIEW_FILE and str(file) not in dryrun.SKIP
|
||||
if file.name == VIEW_FILE and str(file) not in DryRun.skipped_files()
|
||||
]
|
||||
for view_file in view_files:
|
||||
dataset = view_file.parent.parent.name
|
||||
|
|
|
|||
|
|
@ -210,7 +210,7 @@ def respect_dryrun_skip_option(default=True):
|
|||
flags = {True: "--respect-dryrun-skip", False: "--ignore-dryrun-skip"}
|
||||
return click.option(
|
||||
f"{flags[True]}/{flags[False]}",
|
||||
help="Respect or ignore dry run SKIP configuration. "
|
||||
help="Respect or ignore dry run skip configuration. "
|
||||
f"Default is {flags[default]}.",
|
||||
default=default,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ from ..cli.utils import (
|
|||
sql_dir_option,
|
||||
use_cloud_function_option,
|
||||
)
|
||||
from ..dryrun import SKIP as DRYRUN_SKIP
|
||||
from ..dryrun import DryRun
|
||||
from ..metadata.parse_metadata import METADATA_FILE, Metadata
|
||||
from ..util.bigquery_id import sql_table_id
|
||||
from ..util.client_queue import ClientQueue
|
||||
|
|
@ -228,7 +228,7 @@ def publish(
|
|||
|
||||
views = _collect_views(name, sql_dir, project_id, user_facing_only, skip_authorized)
|
||||
if respect_dryrun_skip:
|
||||
views = [view for view in views if view.path not in DRYRUN_SKIP]
|
||||
views = [view for view in views if view.path not in DryRun.skipped_files()]
|
||||
if add_managed_label:
|
||||
for view in views:
|
||||
view.labels["managed"] = ""
|
||||
|
|
|
|||
|
|
@ -0,0 +1,41 @@
|
|||
"""bqetl_project.yaml config loader."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).parent.parent
|
||||
BQETL_PROJECT_CONFIG = "bqetl_project.yaml"
|
||||
|
||||
|
||||
class _ConfigLoader:
|
||||
project_dir: Path = ROOT
|
||||
config_file: str = BQETL_PROJECT_CONFIG
|
||||
|
||||
@property
|
||||
def config(self):
|
||||
config = getattr(self, "_config", None)
|
||||
if config:
|
||||
return config
|
||||
|
||||
self._config = yaml.safe_load((self.project_dir / self.config_file).read_text())
|
||||
return self._config
|
||||
|
||||
def set_project_dir(self, project_dir: Path):
|
||||
"""Update the project root directory."""
|
||||
self.project_dir = project_dir
|
||||
|
||||
def get(self, *args, fallback=None):
|
||||
"""Get the config option specified by args."""
|
||||
conf = self.config
|
||||
|
||||
for arg in args:
|
||||
if arg in conf:
|
||||
conf = conf[arg]
|
||||
else:
|
||||
return fallback
|
||||
|
||||
return conf
|
||||
|
||||
|
||||
ConfigLoader = _ConfigLoader()
|
||||
|
|
@ -17,11 +17,13 @@ import re
|
|||
from enum import Enum
|
||||
from os.path import basename, dirname, exists
|
||||
from pathlib import Path
|
||||
from typing import Set
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
import click
|
||||
from google.cloud import bigquery
|
||||
|
||||
from .config import ConfigLoader
|
||||
from .metadata.parse_metadata import Metadata
|
||||
from .util.common import render
|
||||
|
||||
|
|
@ -31,262 +33,6 @@ except ImportError:
|
|||
# python 3.7 compatibility
|
||||
from backports.cached_property import cached_property # type: ignore
|
||||
|
||||
TEST_PROJECT = "bigquery-etl-integration-test"
|
||||
SKIP = {
|
||||
# Access Denied
|
||||
"sql/moz-fx-data-shared-prod/account_ecosystem_derived/ecosystem_client_id_lookup_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/account_ecosystem_derived/desktop_clients_daily_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/account_ecosystem_restricted/ecosystem_client_id_deletion_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/account_ecosystem_derived/fxa_logging_users_daily_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/activity_stream/impression_stats_flat/view.sql",
|
||||
"sql/moz-fx-data-shared-prod/activity_stream/tile_id_types/view.sql",
|
||||
"sql/moz-fx-data-shared-prod/monitoring_derived/deletion_request_volume_v1/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/monitoring_derived/schema_error_counts_v1/view.sql",
|
||||
"sql/moz-fx-data-shared-prod/monitoring_derived/schema_error_counts_v2/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/monitoring_derived/suggest_impression_rate_v1/init.sql",
|
||||
"sql/moz-fx-data-shared-prod/monitoring_derived/suggest_impression_rate_v1/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/monitoring/schema_error_counts_v1/view.sql",
|
||||
"sql/moz-fx-data-shared-prod/monitoring_derived/structured_error_counts_v1/view.sql",
|
||||
"sql/moz-fx-data-shared-prod/monitoring/structured_error_counts/view.sql",
|
||||
"sql/moz-fx-data-shared-prod/monitoring_derived/telemetry_missing_columns_v1/view.sql",
|
||||
"sql/moz-fx-data-shared-prod/monitoring/telemetry_missing_columns_v1/view.sql",
|
||||
"sql/moz-fx-data-shared-prod/monitoring_derived/telemetry_missing_columns_v2/view.sql",
|
||||
"sql/moz-fx-data-shared-prod/monitoring/telemetry_missing_columns_v2/view.sql",
|
||||
*glob.glob(
|
||||
"sql/moz-fx-data-shared-prod/monitoring*/suggest*_rate*_live*/*.sql",
|
||||
recursive=True,
|
||||
),
|
||||
*glob.glob(
|
||||
"sql/moz-fx-data-shared-prod/monitoring*/topsites*_rate*_live*/*.sql",
|
||||
recursive=True,
|
||||
),
|
||||
"sql/moz-fx-data-shared-prod/pocket/pocket_reach_mau/view.sql",
|
||||
"sql/moz-fx-data-shared-prod/telemetry/buildhub2/view.sql",
|
||||
"sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_content_events_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_auth_bounce_events_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_auth_events_v1/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_delete_events_v1/init.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_delete_events_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_oauth_events_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_auth_events_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_content_events_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_first_seen_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_last_seen_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_amplitude_export_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_amplitude_user_ids_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_amplitude_user_ids_v1/init.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_stdout_events_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/firefox_accounts_derived/nonprod_fxa_auth_events_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/firefox_accounts_derived/nonprod_fxa_content_events_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/firefox_accounts_derived/nonprod_fxa_stdout_events_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/firefox_accounts_derived/nonprod_fxa_server_events_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/firefox_accounts_derived/docker_fxa_admin_server_sanitized_v1/init.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/firefox_accounts_derived/docker_fxa_admin_server_sanitized_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/firefox_accounts_derived/docker_fxa_customs_sanitized_v1/init.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/firefox_accounts_derived/docker_fxa_customs_sanitized_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/fivetran_costs_derived/destinations_v1/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/fivetran_costs_derived/incremental_mar_v1/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_costs_v1/query.sql",
|
||||
*glob.glob(
|
||||
"sql/**/apple_ads_external*/**/query.sql",
|
||||
recursive=True,
|
||||
),
|
||||
"sql/moz-fx-data-shared-prod/regrets_reporter/regrets_reporter_update/view.sql",
|
||||
"sql/moz-fx-data-shared-prod/revenue_derived/client_ltv_v1/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/monitoring/payload_bytes_decoded_all/view.sql",
|
||||
"sql/moz-fx-data-shared-prod/monitoring/payload_bytes_decoded_structured/view.sql",
|
||||
"sql/moz-fx-data-shared-prod/monitoring/payload_bytes_decoded_stub_installer/view.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/monitoring/payload_bytes_decoded_telemetry/view.sql",
|
||||
"sql/moz-fx-data-shared-prod/monitoring/payload_bytes_error_structured/view.sql",
|
||||
"sql/moz-fx-data-shared-prod/monitoring/payload_bytes_error_all/view.sql",
|
||||
"sql/moz-fx-data-shared-prod/monitoring_derived/shredder_progress/view.sql",
|
||||
"sql/moz-fx-data-shared-prod/monitoring/shredder_progress/view.sql",
|
||||
"sql/moz-fx-data-shared-prod/monitoring_derived/telemetry_distinct_docids_v1/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/revenue_derived/client_ltv_normalized/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/revenue_derived/client_ltv_normalized_v1/query.sql",
|
||||
*glob.glob("sql/moz-fx-data-shared-prod/stripe_derived/**/*.sql", recursive=True),
|
||||
*glob.glob("sql/moz-fx-data-shared-prod/stripe_external/**/*.sql", recursive=True),
|
||||
"sql/moz-fx-cjms-nonprod-9a36/cjms_bigquery/flows_live/view.sql",
|
||||
"sql/moz-fx-cjms-nonprod-9a36/cjms_bigquery/flows_v1/query.sql",
|
||||
"sql/moz-fx-cjms-nonprod-9a36/cjms_bigquery/refunds_v1/query.sql",
|
||||
"sql/moz-fx-cjms-nonprod-9a36/cjms_bigquery/subscriptions_v1/query.sql",
|
||||
"sql/moz-fx-cjms-prod-f3c7/cjms_bigquery/flows_v1/query.sql",
|
||||
"sql/moz-fx-cjms-prod-f3c7/cjms_bigquery/refunds_v1/query.sql",
|
||||
"sql/moz-fx-cjms-prod-f3c7/cjms_bigquery/subscriptions_v1/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/subscription_platform_derived/apple_subscriptions_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/subscription_platform_derived/google_subscriptions_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/subscription_platform_derived/nonprod_apple_subscriptions_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/subscription_platform_derived/nonprod_google_subscriptions_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/subscription_platform_derived/nonprod_stripe_subscriptions_history_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_subscriptions_changelog_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_subscriptions_history_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_subscriptions_revised_changelog_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/stripe/itemized_payout_reconciliation/view.sql",
|
||||
"sql/moz-fx-data-shared-prod/mozilla_vpn_derived/add_device_events_v1/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/mozilla_vpn_derived/devices_v1/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/mozilla_vpn_derived/fxa_attribution_v1/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/mozilla_vpn_derived/funnel_product_page_to_subscribed_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/mozilla_vpn_derived/guardian_apple_events_v1/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/mozilla_vpn_derived/login_flows_v1/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/mozilla_vpn_derived/protected_v1/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/mozilla_vpn_derived/site_metrics_empty_check_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/mozilla_vpn_derived/site_metrics_summary_v1/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/mozilla_vpn_derived/subscriptions_v1/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/mozilla_vpn_derived/users_v1/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/mozilla_vpn_external/devices_v1/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/mozilla_vpn_external/subscriptions_v1/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/mozilla_vpn_external/users_v1/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/monitoring_derived/telemetry_missing_columns_v3/query.sql",
|
||||
"sql/moz-fx-data-experiments/monitoring/query_cost_v1/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/mozilla_vpn_external/subscriptions_v1/init.sql",
|
||||
"sql/moz-fx-data-shared-prod/mozilla_vpn_external/users_v1/init.sql",
|
||||
"sql/moz-fx-data-shared-prod/mozilla_vpn_derived/protected_v1/init.sql",
|
||||
"sql/moz-fx-data-shared-prod/mozilla_vpn_derived/add_device_events_v1/init.sql",
|
||||
"sql/moz-fx-data-shared-prod/mozilla_vpn_external/devices_v1/init.sql",
|
||||
"sql/moz-fx-data-shared-prod/fenix_derived/google_ads_campaign_cost_breakdowns_v1/query.sql",
|
||||
*glob.glob("sql/moz-fx-data-shared-prod/search_terms*/**/*.sql", recursive=True),
|
||||
"sql/moz-fx-data-bq-performance/release_criteria/dashboard_health_v1/query.sql",
|
||||
"sql/moz-fx-data-bq-performance/release_criteria/rc_flattened_test_data_v1/query.sql",
|
||||
"sql/moz-fx-data-bq-performance/release_criteria/release_criteria_summary_v1/query.sql",
|
||||
"sql/moz-fx-data-bq-performance/release_criteria/stale_tests_v1/query.sql",
|
||||
"sql/moz-fx-data-bq-performance/release_criteria/release_criteria_v1/query.sql",
|
||||
*glob.glob(
|
||||
"sql/moz-fx-data-shared-prod/contextual_services/**/*.sql", recursive=True
|
||||
),
|
||||
*glob.glob(
|
||||
"sql/moz-fx-data-shared-prod/contextual_services_derived/**/*.sql",
|
||||
recursive=True,
|
||||
),
|
||||
*glob.glob(
|
||||
"sql/moz-fx-data-shared-prod/**/topsites_impression/view.sql", recursive=True
|
||||
),
|
||||
"sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/init.sql",
|
||||
"sql/moz-fx-data-shared-prod/contextual_services_derived/adm_forecasting_v1/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/regrets_reporter/regrets_reporter_summary/view.sql",
|
||||
"sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_tiles_v2/query.sql",
|
||||
*glob.glob(
|
||||
"sql/moz-fx-data-shared-prod/regrets_reporter_derived/regrets_reporter_summary_v1/*.sql", # noqa E501
|
||||
recursive=True,
|
||||
),
|
||||
*glob.glob(
|
||||
"sql/moz-fx-data-shared-prod/mlhackweek_search/**/*.sql", recursive=True
|
||||
),
|
||||
*glob.glob(
|
||||
"sql/moz-fx-data-shared-prod/regrets_reporter_ucs/**/*.sql", recursive=True
|
||||
),
|
||||
"sql/moz-fx-data-shared-prod/telemetry/xfocsp_error_report/view.sql",
|
||||
"sql/moz-fx-data-shared-prod/telemetry/regrets_reporter_update/view.sql",
|
||||
*glob.glob(
|
||||
"sql/moz-fx-data-marketing-prod/acoustic/**/*.sql",
|
||||
recursive=True,
|
||||
),
|
||||
*glob.glob(
|
||||
"sql/moz-fx-data-marketing-prod/adjust_derived/**/*.sql",
|
||||
recursive=True,
|
||||
),
|
||||
*glob.glob(
|
||||
"sql/moz-fx-data-shared-prod/monitoring_derived/airflow_*/*.sql",
|
||||
recursive=True,
|
||||
), # noqa E501
|
||||
*glob.glob(
|
||||
"sql/moz-fx-data-shared-prod/google_ads_derived/**/*.sql",
|
||||
recursive=True,
|
||||
),
|
||||
*glob.glob(
|
||||
"sql/moz-fx-data-shared-prod/**/client_deduplication*/*.sql",
|
||||
recursive=True,
|
||||
),
|
||||
"sql/moz-fx-data-marketing-prod/ga_derived/downloads_with_attribution_v1/query.sql",
|
||||
"sql/moz-fx-data-marketing-prod/ga_derived/downloads_with_attribution_v2/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/fenix_external/installs_by_country_v1/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/fenix/installs_by_country/view.sql",
|
||||
"sql/moz-fx-data-shared-prod/firefox_desktop/top_sites/view.sql",
|
||||
"sql/moz-fx-data-shared-prod/firefox_desktop/quick_suggest/view.sql",
|
||||
# Materialized views
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/experiment_search_events_live_v1/init.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/experiment_events_live_v1/init.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/experiment_search_events_live_v1/init.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/experiment_events_live_v1/init.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/org_mozilla_firefox_derived/experiment_search_events_live_v1/init.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/org_mozilla_firefox_derived/experiment_events_live_v1/init.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/org_mozilla_firefox_beta_derived/experiment_search_events_live_v1/init.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/org_mozilla_firefox_beta_derived/experiment_events_live_v1/init.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/experiment_enrollment_cumulative_population_estimate_v1/view.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/telemetry/experiment_enrollment_cumulative_population_estimate/view.sql", # noqa E501
|
||||
# Already exists (and lacks an "OR REPLACE" clause)
|
||||
"sql/moz-fx-data-shared-prod/org_mozilla_firefox_derived/clients_first_seen_v1/init.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/org_mozilla_firefox_derived/clients_last_seen_v1/init.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/clients_last_seen_v1/init.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/org_mozilla_vrbrowser_derived/clients_last_seen_v1/init.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/core_clients_last_seen_v1/init.sql",
|
||||
"sql/moz-fx-data-shared-prod/telemetry/fxa_users_last_seen_raw_v1/init.sql",
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/core_clients_first_seen_v1/init.sql",
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/fxa_users_services_last_seen_v1/init.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/messaging_system_derived/cfr_users_last_seen_v1/init.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/messaging_system_derived/onboarding_users_last_seen_v1/init.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/messaging_system_derived/snippets_users_last_seen_v1/init.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/messaging_system_derived/whats_new_panel_users_last_seen_v1/init.sql", # noqa E501
|
||||
# Reference table not found
|
||||
"sql/moz-fx-data-shared-prod/monitoring_derived/structured_detailed_error_counts_v1/view.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/monitoring/structured_detailed_error_counts/view.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/org_mozilla_firefox_derived/migrated_clients_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/org_mozilla_firefox_derived/incline_executive_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/org_mozilla_firefox/migrated_clients/view.sql",
|
||||
"sql/moz-fx-data-shared-prod/telemetry/sponsored_tiles_ad_request_fill/view.sql",
|
||||
# No matching signature for function IF
|
||||
"sql/moz-fx-data-shared-prod/static/fxa_amplitude_export_users_last_seen/query.sql",
|
||||
# Duplicate UDF
|
||||
"sql/moz-fx-data-shared-prod/static/fxa_amplitude_export_users_daily/query.sql",
|
||||
# Syntax error
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/clients_last_seen_v1/init.sql", # noqa E501
|
||||
# HTTP Error 408: Request Time-out
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/clients_last_seen_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/latest_versions/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/italy_covid19_outage_v1/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/main_nightly_v1/init.sql",
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/main_nightly_v1/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/main_1pct_v1/init.sql",
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/main_1pct_v1/query.sql",
|
||||
# Query parameter not found
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/experiments_v1/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_scalar_aggregates_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_keyed_scalar_aggregates_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_keyed_boolean_aggregates_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_histogram_aggregates_content_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_histogram_aggregates_gpu_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_histogram_aggregates_parent_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_keyed_histogram_aggregates_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/clients_histogram_aggregates_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/clients_histogram_aggregates_v2/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/clients_histogram_bucket_counts_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/glam_client_probe_counts_extract_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/glam_sample_counts_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/glam_user_counts_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/scalar_percentiles_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/telemetry_derived/clients_scalar_probe_counts_v1/query.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_etl_sql_run_check_v1/query.sql", # noqa E501
|
||||
# Dataset sql/glam-fenix-dev:glam_etl was not found
|
||||
*glob.glob("sql/glam-fenix-dev/glam_etl/**/*.sql", recursive=True),
|
||||
# Query templates
|
||||
"sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/fenix_metrics.template.sql", # noqa E501
|
||||
"sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/mobile_search_clients_daily.template.sql", # noqa E501
|
||||
# Temporary table does not exist
|
||||
# TODO: remove this in a follow up PR
|
||||
*glob.glob(
|
||||
"sql/moz-fx-data-shared-prod/*/baseline_clients_first_seen_v1/*.sql",
|
||||
recursive=True,
|
||||
),
|
||||
# Query too complex
|
||||
"sql/moz-fx-data-shared-prod/firefox_accounts_derived/event_types_history_v1/query.sql",
|
||||
"sql/moz-fx-data-shared-prod/firefox_accounts_derived/event_types_history_v1/init.sql",
|
||||
# Tests
|
||||
"sql/moz-fx-data-test-project/test/simple_view/view.sql",
|
||||
}
|
||||
|
||||
|
||||
class Errors(Enum):
|
||||
"""DryRun errors that require special handling."""
|
||||
|
|
@ -299,12 +45,6 @@ class Errors(Enum):
|
|||
class DryRun:
|
||||
"""Dry run SQL files."""
|
||||
|
||||
# todo: support different dry run endpoints for different projects
|
||||
DRY_RUN_URL = (
|
||||
"https://us-central1-moz-fx-data-shared-prod.cloudfunctions.net/"
|
||||
"bigquery-etl-dryrun"
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
sqlfile,
|
||||
|
|
@ -321,14 +61,43 @@ class DryRun:
|
|||
self.use_cloud_function = use_cloud_function
|
||||
self.client = client if use_cloud_function or client else bigquery.Client()
|
||||
self.respect_skip = respect_skip
|
||||
self.dry_run_url = ConfigLoader.get("dry_run", "function")
|
||||
try:
|
||||
self.metadata = Metadata.of_query_file(self.sqlfile)
|
||||
except FileNotFoundError:
|
||||
self.metadata = None
|
||||
|
||||
@staticmethod
|
||||
def skipped_files() -> Set[str]:
|
||||
"""Return files skipped by dry run."""
|
||||
skip_files = {
|
||||
file
|
||||
for skip in ConfigLoader.get("dry_run", "skip", fallback=[])
|
||||
for file in glob.glob(
|
||||
skip,
|
||||
recursive=True,
|
||||
)
|
||||
}
|
||||
|
||||
# update skip list to include renamed queries in stage.
|
||||
test_project = ConfigLoader.get("dry_run", "test_project", fallback="")
|
||||
file_pattern_re = re.compile(r"sql/([^\/]+)/([^/]+)(/?.*|$)")
|
||||
skip_files.update(
|
||||
[
|
||||
file
|
||||
for skip in ConfigLoader.get("dry_run", "skip", fallback=[])
|
||||
for file in glob.glob(
|
||||
file_pattern_re.sub(f"sql/{test_project}/\\2*\\3", skip),
|
||||
recursive=True,
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
return skip_files
|
||||
|
||||
def skip(self):
|
||||
"""Determine if dry run should be skipped."""
|
||||
return self.respect_skip and self.sqlfile in SKIP
|
||||
return self.respect_skip and self.sqlfile in self.skipped_files()
|
||||
|
||||
def get_sql(self):
|
||||
"""Get SQL content."""
|
||||
|
|
@ -385,7 +154,7 @@ class DryRun:
|
|||
if self.use_cloud_function:
|
||||
r = urlopen(
|
||||
Request(
|
||||
self.DRY_RUN_URL,
|
||||
self.dry_run_url,
|
||||
headers={"Content-Type": "application/json"},
|
||||
data=json.dumps(
|
||||
{
|
||||
|
|
@ -683,21 +452,3 @@ def find_next_word(target, source):
|
|||
if w == target:
|
||||
# get the next word, and remove quotations from column name
|
||||
return split[i + 1].replace("'", "")
|
||||
|
||||
|
||||
def add_test_project_to_skip(sql_dir="sql", project=TEST_PROJECT):
|
||||
"""Update skip list to include renamed queries in stage."""
|
||||
SKIP.update(
|
||||
[
|
||||
p
|
||||
for f in [Path(s) for s in SKIP]
|
||||
for p in glob.glob(
|
||||
f"sql/{TEST_PROJECT}/{f.parent.parent.name}*/{f.parent.name}/{f.name}",
|
||||
recursive=True,
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
# detect test files on startup
|
||||
add_test_project_to_skip()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,215 @@
|
|||
dry_run:
|
||||
function: https://us-central1-moz-fx-data-shared-prod.cloudfunctions.net/bigquery-etl-dryrun
|
||||
test_project: bigquery-etl-integration-test
|
||||
skip:
|
||||
# Access Denied
|
||||
- sql/moz-fx-data-shared-prod/account_ecosystem_derived/ecosystem_client_id_lookup_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/account_ecosystem_derived/desktop_clients_daily_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/account_ecosystem_restricted/ecosystem_client_id_deletion_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/account_ecosystem_derived/fxa_logging_users_daily_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/activity_stream/impression_stats_flat/view.sql
|
||||
- sql/moz-fx-data-shared-prod/activity_stream/tile_id_types/view.sql
|
||||
- sql/moz-fx-data-shared-prod/monitoring_derived/deletion_request_volume_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/monitoring_derived/schema_error_counts_v1/view.sql
|
||||
- sql/moz-fx-data-shared-prod/monitoring_derived/schema_error_counts_v2/query.sql
|
||||
- sql/moz-fx-data-shared-prod/monitoring_derived/suggest_impression_rate_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/monitoring_derived/suggest_impression_rate_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/monitoring/schema_error_counts_v1/view.sql
|
||||
- sql/moz-fx-data-shared-prod/monitoring_derived/structured_error_counts_v1/view.sql
|
||||
- sql/moz-fx-data-shared-prod/monitoring/structured_error_counts/view.sql
|
||||
- sql/moz-fx-data-shared-prod/monitoring_derived/telemetry_missing_columns_v1/view.sql
|
||||
- sql/moz-fx-data-shared-prod/monitoring/telemetry_missing_columns_v1/view.sql
|
||||
- sql/moz-fx-data-shared-prod/monitoring_derived/telemetry_missing_columns_v2/view.sql
|
||||
- sql/moz-fx-data-shared-prod/monitoring/telemetry_missing_columns_v2/view.sql
|
||||
- sql/moz-fx-data-shared-prod/monitoring*/suggest*_rate*_live*/*.sql
|
||||
- sql/moz-fx-data-shared-prod/monitoring*/topsites*_rate*_live*/*.sql
|
||||
- sql/moz-fx-data-shared-prod/pocket/pocket_reach_mau/view.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry/buildhub2/view.sql
|
||||
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_content_events_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_auth_bounce_events_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_auth_events_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_delete_events_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_delete_events_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_oauth_events_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_auth_events_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_content_events_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_first_seen_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_last_seen_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_amplitude_export_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_amplitude_user_ids_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_amplitude_user_ids_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_stdout_events_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/nonprod_fxa_auth_events_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/nonprod_fxa_content_events_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/nonprod_fxa_stdout_events_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/nonprod_fxa_server_events_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/docker_fxa_admin_server_sanitized_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/docker_fxa_admin_server_sanitized_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/docker_fxa_customs_sanitized_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/docker_fxa_customs_sanitized_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/fivetran_costs_derived/destinations_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/fivetran_costs_derived/incremental_mar_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_costs_v1/query.sql
|
||||
- sql/**/apple_ads_external*/**/query.sql
|
||||
- sql/moz-fx-data-shared-prod/regrets_reporter/regrets_reporter_update/view.sql
|
||||
- sql/moz-fx-data-shared-prod/revenue_derived/client_ltv_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/monitoring/payload_bytes_decoded_all/view.sql
|
||||
- sql/moz-fx-data-shared-prod/monitoring/payload_bytes_decoded_structured/view.sql
|
||||
- sql/moz-fx-data-shared-prod/monitoring/payload_bytes_decoded_stub_installer/view.sql
|
||||
- sql/moz-fx-data-shared-prod/monitoring/payload_bytes_decoded_telemetry/view.sql
|
||||
- sql/moz-fx-data-shared-prod/monitoring/payload_bytes_error_structured/view.sql
|
||||
- sql/moz-fx-data-shared-prod/monitoring/payload_bytes_error_all/view.sql
|
||||
- sql/moz-fx-data-shared-prod/monitoring_derived/shredder_progress/view.sql
|
||||
- sql/moz-fx-data-shared-prod/monitoring/shredder_progress/view.sql
|
||||
- sql/moz-fx-data-shared-prod/monitoring_derived/telemetry_distinct_docids_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/revenue_derived/client_ltv_normalized/query.sql
|
||||
- sql/moz-fx-data-shared-prod/revenue_derived/client_ltv_normalized_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/stripe_derived/**/*.sql,
|
||||
- sql/moz-fx-data-shared-prod/stripe_external/**/*.sql,
|
||||
- sql/moz-fx-cjms-nonprod-9a36/cjms_bigquery/flows_live/view.sql
|
||||
- sql/moz-fx-cjms-nonprod-9a36/cjms_bigquery/flows_v1/query.sql
|
||||
- sql/moz-fx-cjms-nonprod-9a36/cjms_bigquery/refunds_v1/query.sql
|
||||
- sql/moz-fx-cjms-nonprod-9a36/cjms_bigquery/subscriptions_v1/query.sql
|
||||
- sql/moz-fx-cjms-prod-f3c7/cjms_bigquery/flows_v1/query.sql
|
||||
- sql/moz-fx-cjms-prod-f3c7/cjms_bigquery/refunds_v1/query.sql
|
||||
- sql/moz-fx-cjms-prod-f3c7/cjms_bigquery/subscriptions_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/subscription_platform_derived/apple_subscriptions_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/subscription_platform_derived/google_subscriptions_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/subscription_platform_derived/nonprod_apple_subscriptions_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/subscription_platform_derived/nonprod_google_subscriptions_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/subscription_platform_derived/nonprod_stripe_subscriptions_history_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_subscriptions_revised_changelog_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_subscriptions_changelog_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_subscriptions_history_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/stripe/itemized_payout_reconciliation/view.sql
|
||||
- sql/moz-fx-data-shared-prod/mozilla_vpn_derived/add_device_events_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/mozilla_vpn_derived/devices_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/mozilla_vpn_derived/fxa_attribution_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/mozilla_vpn_derived/funnel_product_page_to_subscribed_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/mozilla_vpn_derived/guardian_apple_events_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/mozilla_vpn_derived/login_flows_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/mozilla_vpn_derived/protected_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/mozilla_vpn_derived/site_metrics_empty_check_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/mozilla_vpn_derived/site_metrics_summary_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/mozilla_vpn_derived/subscriptions_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/mozilla_vpn_derived/users_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/mozilla_vpn_external/devices_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/mozilla_vpn_external/subscriptions_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/mozilla_vpn_external/users_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/monitoring_derived/telemetry_missing_columns_v3/query.sql
|
||||
- sql/moz-fx-data-experiments/monitoring/query_cost_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/mozilla_vpn_external/subscriptions_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/mozilla_vpn_external/users_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/mozilla_vpn_derived/protected_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/mozilla_vpn_derived/add_device_events_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/mozilla_vpn_external/devices_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/fenix_derived/google_ads_campaign_cost_breakdowns_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/search_terms*/**/*.sql
|
||||
- sql/moz-fx-data-bq-performance/release_criteria/dashboard_health_v1/query.sql
|
||||
- sql/moz-fx-data-bq-performance/release_criteria/rc_flattened_test_data_v1/query.sql
|
||||
- sql/moz-fx-data-bq-performance/release_criteria/release_criteria_summary_v1/query.sql
|
||||
- sql/moz-fx-data-bq-performance/release_criteria/stale_tests_v1/query.sql
|
||||
- sql/moz-fx-data-bq-performance/release_criteria/release_criteria_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/contextual_services/**/*.sql
|
||||
- sql/moz-fx-data-shared-prod/contextual_services_derived/**/*.sql
|
||||
- sql/moz-fx-data-shared-prod/**/topsites_impression/view.sql
|
||||
- sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/contextual_services_derived/adm_forecasting_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/regrets_reporter/regrets_reporter_summary/view.sql
|
||||
- sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql
|
||||
- sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_tiles_v2/query.sql
|
||||
- sql/moz-fx-data-shared-prod/regrets_reporter_derived/regrets_reporter_summary_v1/*.sql
|
||||
- sql/moz-fx-data-shared-prod/mlhackweek_search/**/*.sql
|
||||
- sql/moz-fx-data-shared-prod/regrets_reporter_ucs/**/*.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry/xfocsp_error_report/view.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry/regrets_reporter_update/view.sql
|
||||
- sql/moz-fx-data-marketing-prod/acoustic/**/*.sql
|
||||
- sql/moz-fx-data-marketing-prod/adjust_derived/**/*.sql
|
||||
- sql/moz-fx-data-shared-prod/monitoring_derived/airflow_*/*.sql
|
||||
- sql/moz-fx-data-shared-prod/google_ads_derived/**/*.sql
|
||||
- sql/moz-fx-data-shared-prod/**/client_deduplication*/*.sql
|
||||
- sql/moz-fx-data-marketing-prod/ga_derived/downloads_with_attribution_v1/query.sql
|
||||
- sql/moz-fx-data-marketing-prod/ga_derived/downloads_with_attribution_v2/query.sql
|
||||
- sql/moz-fx-data-shared-prod/fenix_external/installs_by_country_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/fenix/installs_by_country/view.sql
|
||||
- sql/moz-fx-data-shared-prod/firefox_desktop/top_sites/view.sql
|
||||
- sql/moz-fx-data-shared-prod/firefox_desktop/quick_suggest/view.sql
|
||||
# Materialized views
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/experiment_search_events_live_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/experiment_events_live_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/experiment_search_events_live_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/experiment_events_live_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/org_mozilla_firefox_derived/experiment_search_events_live_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/org_mozilla_firefox_derived/experiment_events_live_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/org_mozilla_firefox_beta_derived/experiment_search_events_live_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/org_mozilla_firefox_beta_derived/experiment_events_live_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/experiment_enrollment_cumulative_population_estimate_v1/view.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry/experiment_enrollment_cumulative_population_estimate/view.sql
|
||||
# Already exists (and lacks an "OR REPLACE" clause)
|
||||
- sql/moz-fx-data-shared-prod/org_mozilla_firefox_derived/clients_first_seen_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/org_mozilla_firefox_derived/clients_last_seen_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/clients_last_seen_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/org_mozilla_vrbrowser_derived/clients_last_seen_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/core_clients_last_seen_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry/fxa_users_last_seen_raw_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/core_clients_first_seen_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/fxa_users_services_last_seen_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/messaging_system_derived/cfr_users_last_seen_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/messaging_system_derived/onboarding_users_last_seen_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/messaging_system_derived/snippets_users_last_seen_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/messaging_system_derived/whats_new_panel_users_last_seen_v1/init.sql
|
||||
# Reference table not found
|
||||
- sql/moz-fx-data-shared-prod/monitoring_derived/structured_detailed_error_counts_v1/view.sql
|
||||
- sql/moz-fx-data-shared-prod/monitoring/structured_detailed_error_counts/view.sql
|
||||
- sql/moz-fx-data-shared-prod/org_mozilla_firefox_derived/migrated_clients_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/org_mozilla_firefox_derived/incline_executive_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/org_mozilla_firefox/migrated_clients/view.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry/sponsored_tiles_ad_request_fill/view.sql
|
||||
# No matching signature for function IF
|
||||
- sql/moz-fx-data-shared-prod/static/fxa_amplitude_export_users_last_seen/query.sql
|
||||
# Duplicate UDF
|
||||
- sql/moz-fx-data-shared-prod/static/fxa_amplitude_export_users_daily/query.sql
|
||||
# Syntax error
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/clients_last_seen_v1/init.sql
|
||||
# HTTP Error 408: Request Time-out
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/clients_last_seen_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/latest_versions/query.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/italy_covid19_outage_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/main_nightly_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/main_nightly_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/main_1pct_v1/init.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/main_1pct_v1/query.sql
|
||||
# Query parameter not found
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/experiments_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_scalar_aggregates_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_keyed_scalar_aggregates_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_keyed_boolean_aggregates_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_histogram_aggregates_content_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_histogram_aggregates_gpu_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_histogram_aggregates_parent_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_keyed_histogram_aggregates_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/clients_histogram_aggregates_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/clients_histogram_aggregates_v2/query.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/clients_histogram_bucket_counts_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/glam_client_probe_counts_extract_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/glam_sample_counts_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/glam_user_counts_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/scalar_percentiles_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/telemetry_derived/clients_scalar_probe_counts_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_etl_sql_run_check_v1/query.sql
|
||||
# Dataset sql/glam-fenix-dev:glam_etl was not found
|
||||
- sql/glam-fenix-dev/glam_etl/**/*.sql
|
||||
# Query templates
|
||||
- sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/fenix_metrics.template.sql
|
||||
- sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/mobile_search_clients_daily.template.sql
|
||||
# Temporary table does not exist
|
||||
# TODO: remove this in a follow up PR
|
||||
- sql/moz-fx-data-shared-prod/*/baseline_clients_first_seen_v1/*.sql
|
||||
# Query too complex
|
||||
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/event_types_history_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/firefox_accounts_derived/event_types_history_v1/init.sql
|
||||
# Tests
|
||||
- sql/moz-fx-data-test-project/test/simple_view/view.sql
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
dry_run:
|
||||
function: https://us-central1-moz-fx-data-shared-prod.cloudfunctions.net/bigquery-etl-dryrun
|
||||
test_project: bigquery-etl-integration-test
|
||||
skip:
|
||||
# Access Denied
|
||||
- sql/moz-fx-data-shared-prod/test_derived/some_query_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/test_derived/another_query_v1/query.sql
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
from pathlib import Path
|
||||
|
||||
from bigquery_etl.config import ConfigLoader
|
||||
|
||||
TEST_DIR = Path(__file__).parent
|
||||
|
||||
|
||||
class TestConfig:
|
||||
def test_config_loader_set_project(self):
|
||||
ConfigLoader.set_project_dir(TEST_DIR / "data")
|
||||
assert ConfigLoader.project_dir == TEST_DIR / "data"
|
||||
|
||||
def test_config_loader_get(self):
|
||||
ConfigLoader.set_project_dir(TEST_DIR / "data")
|
||||
|
||||
assert "function" in ConfigLoader.get("dry_run")
|
||||
assert "test_project" in ConfigLoader.get("dry_run")
|
||||
assert "skip" in ConfigLoader.get("dry_run")
|
||||
|
||||
assert (
|
||||
ConfigLoader.get("dry_run", "test_project")
|
||||
== "bigquery-etl-integration-test"
|
||||
)
|
||||
assert len(ConfigLoader.get("dry_run", "skip")) == 2
|
||||
|
||||
assert "dry_run" in ConfigLoader.get()
|
||||
|
||||
def test_config_loader_get_non_existing(self):
|
||||
ConfigLoader.set_project_dir(TEST_DIR / "data")
|
||||
|
||||
assert ConfigLoader.get("non_existing") is None
|
||||
assert ConfigLoader.get("dry_run", "non_existing") is None
|
||||
assert ConfigLoader.get("non_existing", fallback=[]) == []
|
||||
assert ConfigLoader.get("dry_run", "foo", fallback=123) == 123
|
||||
Загрузка…
Ссылка в новой задаче