Add ConfigLoader and move dry run skip to bqetl_project.yaml (#4000)

* Add ConfigLoader and move dry run skip to bqetl_project.yaml

* format tests

bigquery_etl/cli/__init__.py

@@ -2,6 +2,7 @@
 
 import logging
 import warnings
+from pathlib import Path
 
 import click
 
@@ -19,6 +20,7 @@ from ..cli.query import query
 from ..cli.routine import mozfun, routine
 from ..cli.stage import stage
 from ..cli.view import view
+from ..config import ConfigLoader
 from ..copy_deduplicate import copy_deduplicate
 from ..dependency import dependency
 from ..docs import docs_
@@ -73,4 +75,5 @@ def cli(prog_name=None):
 
 
 if __name__ == "__main__":
+    ConfigLoader.set_project_dir(Path().absolute())
     cli()

bigquery_etl/cli/dryrun.py

@@ -13,7 +13,7 @@ import click
 from google.cloud import bigquery
 
 from ..cli.utils import is_authenticated
-from ..dryrun import SKIP, DryRun
+from ..dryrun import DryRun
 
 
 @click.command(
@@ -55,7 +55,7 @@ from ..dryrun import SKIP, DryRun
 )
 @click.option(
     "--respect-skip/--ignore-skip",
-    help="Respect or ignore query SKIP configuration. Default is --respect-skip.",
+    help="Respect or ignore query skip configuration. Default is --respect-skip.",
     default=True,
 )
 @click.option(
@@ -71,7 +71,7 @@ def dryrun(
     project: str,
 ):
     """Perform a dry run."""
-    file_names = ("query.sql", "view.sql", "part*.sql", "init.sql")
+    file_names = ("query.sql", "view.sql", "part*.sql", "init.sql", "checks.sql")
     file_re = re.compile("|".join(map(fnmatch.translate, file_names)))
 
     sql_files: Set[str] = set()
@@ -89,7 +89,7 @@ def dryrun(
             click.echo(f"Invalid path {path}", err=True)
             sys.exit(1)
     if respect_skip:
-        sql_files -= SKIP
+        sql_files -= DryRun.skipped_files()
 
     if not sql_files:
         print("Skipping dry run because no queries matched")

bigquery_etl/cli/query.py

@@ -40,7 +40,7 @@ from ..cli.utils import (
     use_cloud_function_option,
 )
 from ..dependency import get_dependency_graph
-from ..dryrun import SKIP, DryRun
+from ..dryrun import DryRun
 from ..format_sql.format import SKIP as SKIP_FORMAT
 from ..format_sql.formatter import reformat
 from ..metadata import validate_metadata
@@ -1569,7 +1569,7 @@ def _update_query_schema(
 
     Return True if the schema changed, False if it is unchanged.
     """
-    if respect_dryrun_skip and str(query_file) in SKIP:
+    if respect_dryrun_skip and str(query_file) in DryRun.skipped_files():
         click.echo(f"{query_file} dry runs are skipped. Cannot update schemas.")
         return
 
@@ -1838,7 +1838,7 @@ def deploy(
         )
 
     def _deploy(query_file):
-        if respect_dryrun_skip and str(query_file) in SKIP:
+        if respect_dryrun_skip and str(query_file) in DryRun.skipped_files():
             click.echo(f"{query_file} dry runs are skipped. Cannot validate schemas.")
             return
 

bigquery_etl/cli/stage.py

@@ -10,12 +10,12 @@ from pathlib import Path
 import click
 from google.cloud import bigquery
 
-from .. import dryrun
 from ..cli.query import deploy as deploy_query_schema
 from ..cli.query import update as update_query_schema
 from ..cli.routine import publish as publish_routine
 from ..cli.utils import paths_matching_name_pattern, sql_dir_option
 from ..cli.view import publish as publish_view
+from ..dryrun import DryRun
 from ..routine.parse_routine import (
     UDF_FILE,
     RawRoutine,
@@ -186,9 +186,6 @@ def deploy(
             if artifact_file.parent.exists():
                 shutil.rmtree(artifact_file.parent)
 
-    # update dryrun skip list
-    dryrun.add_test_project_to_skip(sql_dir, project_id)
-
     # deploy to stage
     _deploy_artifacts(ctx, updated_artifact_files, project_id, dataset_suffix, sql_dir)
 
@@ -368,7 +365,7 @@ def _deploy_artifacts(ctx, artifact_files, project_id, dataset_suffix, sql_dir):
             name=str(query_file),
             sql_dir=sql_dir,
             project_id=project_id,
-            respect_dryrun_skip=False,
+            respect_dryrun_skip=True,
         )
         ctx.invoke(
             deploy_query_schema,
@@ -387,7 +384,7 @@ def _deploy_artifacts(ctx, artifact_files, project_id, dataset_suffix, sql_dir):
     view_files = [
         file
         for file in artifact_files
-        if file.name == VIEW_FILE and str(file) not in dryrun.SKIP
+        if file.name == VIEW_FILE and str(file) not in DryRun.skipped_files()
     ]
     for view_file in view_files:
         dataset = view_file.parent.parent.name

bigquery_etl/cli/utils.py

@@ -210,7 +210,7 @@ def respect_dryrun_skip_option(default=True):
     flags = {True: "--respect-dryrun-skip", False: "--ignore-dryrun-skip"}
     return click.option(
         f"{flags[True]}/{flags[False]}",
-        help="Respect or ignore dry run SKIP configuration. "
+        help="Respect or ignore dry run skip configuration. "
         f"Default is {flags[default]}.",
         default=default,
     )

bigquery_etl/cli/view.py

@@ -19,7 +19,7 @@ from ..cli.utils import (
     sql_dir_option,
     use_cloud_function_option,
 )
-from ..dryrun import SKIP as DRYRUN_SKIP
+from ..dryrun import DryRun
 from ..metadata.parse_metadata import METADATA_FILE, Metadata
 from ..util.bigquery_id import sql_table_id
 from ..util.client_queue import ClientQueue
@@ -228,7 +228,7 @@ def publish(
 
     views = _collect_views(name, sql_dir, project_id, user_facing_only, skip_authorized)
     if respect_dryrun_skip:
-        views = [view for view in views if view.path not in DRYRUN_SKIP]
+        views = [view for view in views if view.path not in DryRun.skipped_files()]
     if add_managed_label:
         for view in views:
             view.labels["managed"] = ""

bigquery_etl/config.py

@@ -0,0 +1,41 @@
+"""bqetl_project.yaml config loader."""
+
+from pathlib import Path
+
+import yaml
+
+ROOT = Path(__file__).parent.parent
+BQETL_PROJECT_CONFIG = "bqetl_project.yaml"
+
+
+class _ConfigLoader:
+    project_dir: Path = ROOT
+    config_file: str = BQETL_PROJECT_CONFIG
+
+    @property
+    def config(self):
+        config = getattr(self, "_config", None)
+        if config:
+            return config
+
+        self._config = yaml.safe_load((self.project_dir / self.config_file).read_text())
+        return self._config
+
+    def set_project_dir(self, project_dir: Path):
+        """Update the project root directory."""
+        self.project_dir = project_dir
+
+    def get(self, *args, fallback=None):
+        """Get the config option specified by args."""
+        conf = self.config
+
+        for arg in args:
+            if arg in conf:
+                conf = conf[arg]
+            else:
+                return fallback
+
+        return conf
+
+
+ConfigLoader = _ConfigLoader()

bigquery_etl/dryrun.py

@@ -17,11 +17,13 @@ import re
 from enum import Enum
 from os.path import basename, dirname, exists
 from pathlib import Path
+from typing import Set
 from urllib.request import Request, urlopen
 
 import click
 from google.cloud import bigquery
 
+from .config import ConfigLoader
 from .metadata.parse_metadata import Metadata
 from .util.common import render
 
@@ -31,262 +33,6 @@ except ImportError:
     # python 3.7 compatibility
     from backports.cached_property import cached_property  # type: ignore
 
-TEST_PROJECT = "bigquery-etl-integration-test"
-SKIP = {
-    # Access Denied
-    "sql/moz-fx-data-shared-prod/account_ecosystem_derived/ecosystem_client_id_lookup_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/account_ecosystem_derived/desktop_clients_daily_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/account_ecosystem_restricted/ecosystem_client_id_deletion_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/account_ecosystem_derived/fxa_logging_users_daily_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/activity_stream/impression_stats_flat/view.sql",
-    "sql/moz-fx-data-shared-prod/activity_stream/tile_id_types/view.sql",
-    "sql/moz-fx-data-shared-prod/monitoring_derived/deletion_request_volume_v1/query.sql",
-    "sql/moz-fx-data-shared-prod/monitoring_derived/schema_error_counts_v1/view.sql",
-    "sql/moz-fx-data-shared-prod/monitoring_derived/schema_error_counts_v2/query.sql",
-    "sql/moz-fx-data-shared-prod/monitoring_derived/suggest_impression_rate_v1/init.sql",
-    "sql/moz-fx-data-shared-prod/monitoring_derived/suggest_impression_rate_v1/query.sql",
-    "sql/moz-fx-data-shared-prod/monitoring/schema_error_counts_v1/view.sql",
-    "sql/moz-fx-data-shared-prod/monitoring_derived/structured_error_counts_v1/view.sql",
-    "sql/moz-fx-data-shared-prod/monitoring/structured_error_counts/view.sql",
-    "sql/moz-fx-data-shared-prod/monitoring_derived/telemetry_missing_columns_v1/view.sql",
-    "sql/moz-fx-data-shared-prod/monitoring/telemetry_missing_columns_v1/view.sql",
-    "sql/moz-fx-data-shared-prod/monitoring_derived/telemetry_missing_columns_v2/view.sql",
-    "sql/moz-fx-data-shared-prod/monitoring/telemetry_missing_columns_v2/view.sql",
-    *glob.glob(
-        "sql/moz-fx-data-shared-prod/monitoring*/suggest*_rate*_live*/*.sql",
-        recursive=True,
-    ),
-    *glob.glob(
-        "sql/moz-fx-data-shared-prod/monitoring*/topsites*_rate*_live*/*.sql",
-        recursive=True,
-    ),
-    "sql/moz-fx-data-shared-prod/pocket/pocket_reach_mau/view.sql",
-    "sql/moz-fx-data-shared-prod/telemetry/buildhub2/view.sql",
-    "sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_content_events_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_auth_bounce_events_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_auth_events_v1/query.sql",
-    "sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_delete_events_v1/init.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_delete_events_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_oauth_events_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_auth_events_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_content_events_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_first_seen_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_last_seen_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_amplitude_export_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_amplitude_user_ids_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_amplitude_user_ids_v1/init.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_stdout_events_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/firefox_accounts_derived/nonprod_fxa_auth_events_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/firefox_accounts_derived/nonprod_fxa_content_events_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/firefox_accounts_derived/nonprod_fxa_stdout_events_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/firefox_accounts_derived/nonprod_fxa_server_events_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/firefox_accounts_derived/docker_fxa_admin_server_sanitized_v1/init.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/firefox_accounts_derived/docker_fxa_admin_server_sanitized_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/firefox_accounts_derived/docker_fxa_customs_sanitized_v1/init.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/firefox_accounts_derived/docker_fxa_customs_sanitized_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/fivetran_costs_derived/destinations_v1/query.sql",
-    "sql/moz-fx-data-shared-prod/fivetran_costs_derived/incremental_mar_v1/query.sql",
-    "sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_costs_v1/query.sql",
-    *glob.glob(
-        "sql/**/apple_ads_external*/**/query.sql",
-        recursive=True,
-    ),
-    "sql/moz-fx-data-shared-prod/regrets_reporter/regrets_reporter_update/view.sql",
-    "sql/moz-fx-data-shared-prod/revenue_derived/client_ltv_v1/query.sql",
-    "sql/moz-fx-data-shared-prod/monitoring/payload_bytes_decoded_all/view.sql",
-    "sql/moz-fx-data-shared-prod/monitoring/payload_bytes_decoded_structured/view.sql",
-    "sql/moz-fx-data-shared-prod/monitoring/payload_bytes_decoded_stub_installer/view.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/monitoring/payload_bytes_decoded_telemetry/view.sql",
-    "sql/moz-fx-data-shared-prod/monitoring/payload_bytes_error_structured/view.sql",
-    "sql/moz-fx-data-shared-prod/monitoring/payload_bytes_error_all/view.sql",
-    "sql/moz-fx-data-shared-prod/monitoring_derived/shredder_progress/view.sql",
-    "sql/moz-fx-data-shared-prod/monitoring/shredder_progress/view.sql",
-    "sql/moz-fx-data-shared-prod/monitoring_derived/telemetry_distinct_docids_v1/query.sql",
-    "sql/moz-fx-data-shared-prod/revenue_derived/client_ltv_normalized/query.sql",
-    "sql/moz-fx-data-shared-prod/revenue_derived/client_ltv_normalized_v1/query.sql",
-    *glob.glob("sql/moz-fx-data-shared-prod/stripe_derived/**/*.sql", recursive=True),
-    *glob.glob("sql/moz-fx-data-shared-prod/stripe_external/**/*.sql", recursive=True),
-    "sql/moz-fx-cjms-nonprod-9a36/cjms_bigquery/flows_live/view.sql",
-    "sql/moz-fx-cjms-nonprod-9a36/cjms_bigquery/flows_v1/query.sql",
-    "sql/moz-fx-cjms-nonprod-9a36/cjms_bigquery/refunds_v1/query.sql",
-    "sql/moz-fx-cjms-nonprod-9a36/cjms_bigquery/subscriptions_v1/query.sql",
-    "sql/moz-fx-cjms-prod-f3c7/cjms_bigquery/flows_v1/query.sql",
-    "sql/moz-fx-cjms-prod-f3c7/cjms_bigquery/refunds_v1/query.sql",
-    "sql/moz-fx-cjms-prod-f3c7/cjms_bigquery/subscriptions_v1/query.sql",
-    "sql/moz-fx-data-shared-prod/subscription_platform_derived/apple_subscriptions_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/subscription_platform_derived/google_subscriptions_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/subscription_platform_derived/nonprod_apple_subscriptions_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/subscription_platform_derived/nonprod_google_subscriptions_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/subscription_platform_derived/nonprod_stripe_subscriptions_history_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_subscriptions_changelog_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_subscriptions_history_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_subscriptions_revised_changelog_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/stripe/itemized_payout_reconciliation/view.sql",
-    "sql/moz-fx-data-shared-prod/mozilla_vpn_derived/add_device_events_v1/query.sql",
-    "sql/moz-fx-data-shared-prod/mozilla_vpn_derived/devices_v1/query.sql",
-    "sql/moz-fx-data-shared-prod/mozilla_vpn_derived/fxa_attribution_v1/query.sql",
-    "sql/moz-fx-data-shared-prod/mozilla_vpn_derived/funnel_product_page_to_subscribed_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/mozilla_vpn_derived/guardian_apple_events_v1/query.sql",
-    "sql/moz-fx-data-shared-prod/mozilla_vpn_derived/login_flows_v1/query.sql",
-    "sql/moz-fx-data-shared-prod/mozilla_vpn_derived/protected_v1/query.sql",
-    "sql/moz-fx-data-shared-prod/mozilla_vpn_derived/site_metrics_empty_check_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/mozilla_vpn_derived/site_metrics_summary_v1/query.sql",
-    "sql/moz-fx-data-shared-prod/mozilla_vpn_derived/subscriptions_v1/query.sql",
-    "sql/moz-fx-data-shared-prod/mozilla_vpn_derived/users_v1/query.sql",
-    "sql/moz-fx-data-shared-prod/mozilla_vpn_external/devices_v1/query.sql",
-    "sql/moz-fx-data-shared-prod/mozilla_vpn_external/subscriptions_v1/query.sql",
-    "sql/moz-fx-data-shared-prod/mozilla_vpn_external/users_v1/query.sql",
-    "sql/moz-fx-data-shared-prod/monitoring_derived/telemetry_missing_columns_v3/query.sql",
-    "sql/moz-fx-data-experiments/monitoring/query_cost_v1/query.sql",
-    "sql/moz-fx-data-shared-prod/mozilla_vpn_external/subscriptions_v1/init.sql",
-    "sql/moz-fx-data-shared-prod/mozilla_vpn_external/users_v1/init.sql",
-    "sql/moz-fx-data-shared-prod/mozilla_vpn_derived/protected_v1/init.sql",
-    "sql/moz-fx-data-shared-prod/mozilla_vpn_derived/add_device_events_v1/init.sql",
-    "sql/moz-fx-data-shared-prod/mozilla_vpn_external/devices_v1/init.sql",
-    "sql/moz-fx-data-shared-prod/fenix_derived/google_ads_campaign_cost_breakdowns_v1/query.sql",
-    *glob.glob("sql/moz-fx-data-shared-prod/search_terms*/**/*.sql", recursive=True),
-    "sql/moz-fx-data-bq-performance/release_criteria/dashboard_health_v1/query.sql",
-    "sql/moz-fx-data-bq-performance/release_criteria/rc_flattened_test_data_v1/query.sql",
-    "sql/moz-fx-data-bq-performance/release_criteria/release_criteria_summary_v1/query.sql",
-    "sql/moz-fx-data-bq-performance/release_criteria/stale_tests_v1/query.sql",
-    "sql/moz-fx-data-bq-performance/release_criteria/release_criteria_v1/query.sql",
-    *glob.glob(
-        "sql/moz-fx-data-shared-prod/contextual_services/**/*.sql", recursive=True
-    ),
-    *glob.glob(
-        "sql/moz-fx-data-shared-prod/contextual_services_derived/**/*.sql",
-        recursive=True,
-    ),
-    *glob.glob(
-        "sql/moz-fx-data-shared-prod/**/topsites_impression/view.sql", recursive=True
-    ),
-    "sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql",
-    "sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/init.sql",
-    "sql/moz-fx-data-shared-prod/contextual_services_derived/adm_forecasting_v1/query.sql",
-    "sql/moz-fx-data-shared-prod/regrets_reporter/regrets_reporter_summary/view.sql",
-    "sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql",
-    "sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_tiles_v2/query.sql",
-    *glob.glob(
-        "sql/moz-fx-data-shared-prod/regrets_reporter_derived/regrets_reporter_summary_v1/*.sql",  # noqa E501
-        recursive=True,
-    ),
-    *glob.glob(
-        "sql/moz-fx-data-shared-prod/mlhackweek_search/**/*.sql", recursive=True
-    ),
-    *glob.glob(
-        "sql/moz-fx-data-shared-prod/regrets_reporter_ucs/**/*.sql", recursive=True
-    ),
-    "sql/moz-fx-data-shared-prod/telemetry/xfocsp_error_report/view.sql",
-    "sql/moz-fx-data-shared-prod/telemetry/regrets_reporter_update/view.sql",
-    *glob.glob(
-        "sql/moz-fx-data-marketing-prod/acoustic/**/*.sql",
-        recursive=True,
-    ),
-    *glob.glob(
-        "sql/moz-fx-data-marketing-prod/adjust_derived/**/*.sql",
-        recursive=True,
-    ),
-    *glob.glob(
-        "sql/moz-fx-data-shared-prod/monitoring_derived/airflow_*/*.sql",
-        recursive=True,
-    ),  # noqa E501
-    *glob.glob(
-        "sql/moz-fx-data-shared-prod/google_ads_derived/**/*.sql",
-        recursive=True,
-    ),
-    *glob.glob(
-        "sql/moz-fx-data-shared-prod/**/client_deduplication*/*.sql",
-        recursive=True,
-    ),
-    "sql/moz-fx-data-marketing-prod/ga_derived/downloads_with_attribution_v1/query.sql",
-    "sql/moz-fx-data-marketing-prod/ga_derived/downloads_with_attribution_v2/query.sql",
-    "sql/moz-fx-data-shared-prod/fenix_external/installs_by_country_v1/query.sql",
-    "sql/moz-fx-data-shared-prod/fenix/installs_by_country/view.sql",
-    "sql/moz-fx-data-shared-prod/firefox_desktop/top_sites/view.sql",
-    "sql/moz-fx-data-shared-prod/firefox_desktop/quick_suggest/view.sql",
-    # Materialized views
-    "sql/moz-fx-data-shared-prod/telemetry_derived/experiment_search_events_live_v1/init.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/telemetry_derived/experiment_events_live_v1/init.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/experiment_search_events_live_v1/init.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/experiment_events_live_v1/init.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/org_mozilla_firefox_derived/experiment_search_events_live_v1/init.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/org_mozilla_firefox_derived/experiment_events_live_v1/init.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/org_mozilla_firefox_beta_derived/experiment_search_events_live_v1/init.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/org_mozilla_firefox_beta_derived/experiment_events_live_v1/init.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/telemetry_derived/experiment_enrollment_cumulative_population_estimate_v1/view.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/telemetry/experiment_enrollment_cumulative_population_estimate/view.sql",  # noqa E501
-    # Already exists (and lacks an "OR REPLACE" clause)
-    "sql/moz-fx-data-shared-prod/org_mozilla_firefox_derived/clients_first_seen_v1/init.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/org_mozilla_firefox_derived/clients_last_seen_v1/init.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/clients_last_seen_v1/init.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/org_mozilla_vrbrowser_derived/clients_last_seen_v1/init.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/telemetry_derived/core_clients_last_seen_v1/init.sql",
-    "sql/moz-fx-data-shared-prod/telemetry/fxa_users_last_seen_raw_v1/init.sql",
-    "sql/moz-fx-data-shared-prod/telemetry_derived/core_clients_first_seen_v1/init.sql",
-    "sql/moz-fx-data-shared-prod/telemetry_derived/fxa_users_services_last_seen_v1/init.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/messaging_system_derived/cfr_users_last_seen_v1/init.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/messaging_system_derived/onboarding_users_last_seen_v1/init.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/messaging_system_derived/snippets_users_last_seen_v1/init.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/messaging_system_derived/whats_new_panel_users_last_seen_v1/init.sql",  # noqa E501
-    # Reference table not found
-    "sql/moz-fx-data-shared-prod/monitoring_derived/structured_detailed_error_counts_v1/view.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/monitoring/structured_detailed_error_counts/view.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/org_mozilla_firefox_derived/migrated_clients_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/org_mozilla_firefox_derived/incline_executive_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/org_mozilla_firefox/migrated_clients/view.sql",
-    "sql/moz-fx-data-shared-prod/telemetry/sponsored_tiles_ad_request_fill/view.sql",
-    # No matching signature for function IF
-    "sql/moz-fx-data-shared-prod/static/fxa_amplitude_export_users_last_seen/query.sql",
-    # Duplicate UDF
-    "sql/moz-fx-data-shared-prod/static/fxa_amplitude_export_users_daily/query.sql",
-    # Syntax error
-    "sql/moz-fx-data-shared-prod/telemetry_derived/clients_last_seen_v1/init.sql",  # noqa E501
-    # HTTP Error 408: Request Time-out
-    "sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/telemetry_derived/clients_last_seen_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/telemetry_derived/latest_versions/query.sql",
-    "sql/moz-fx-data-shared-prod/telemetry_derived/italy_covid19_outage_v1/query.sql",
-    "sql/moz-fx-data-shared-prod/telemetry_derived/main_nightly_v1/init.sql",
-    "sql/moz-fx-data-shared-prod/telemetry_derived/main_nightly_v1/query.sql",
-    "sql/moz-fx-data-shared-prod/telemetry_derived/main_1pct_v1/init.sql",
-    "sql/moz-fx-data-shared-prod/telemetry_derived/main_1pct_v1/query.sql",
-    # Query parameter not found
-    "sql/moz-fx-data-shared-prod/telemetry_derived/experiments_v1/query.sql",
-    "sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_scalar_aggregates_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_keyed_scalar_aggregates_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_keyed_boolean_aggregates_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_histogram_aggregates_content_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_histogram_aggregates_gpu_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_histogram_aggregates_parent_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_keyed_histogram_aggregates_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/telemetry_derived/clients_histogram_aggregates_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/telemetry_derived/clients_histogram_aggregates_v2/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/telemetry_derived/clients_histogram_bucket_counts_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/telemetry_derived/glam_client_probe_counts_extract_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/telemetry_derived/glam_sample_counts_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/telemetry_derived/glam_user_counts_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/telemetry_derived/scalar_percentiles_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/telemetry_derived/clients_scalar_probe_counts_v1/query.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_etl_sql_run_check_v1/query.sql",  # noqa E501
-    # Dataset sql/glam-fenix-dev:glam_etl was not found
-    *glob.glob("sql/glam-fenix-dev/glam_etl/**/*.sql", recursive=True),
-    # Query templates
-    "sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/fenix_metrics.template.sql",  # noqa E501
-    "sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/mobile_search_clients_daily.template.sql",  # noqa E501
-    # Temporary table does not exist
-    # TODO: remove this in a follow up PR
-    *glob.glob(
-        "sql/moz-fx-data-shared-prod/*/baseline_clients_first_seen_v1/*.sql",
-        recursive=True,
-    ),
-    # Query too complex
-    "sql/moz-fx-data-shared-prod/firefox_accounts_derived/event_types_history_v1/query.sql",
-    "sql/moz-fx-data-shared-prod/firefox_accounts_derived/event_types_history_v1/init.sql",
-    # Tests
-    "sql/moz-fx-data-test-project/test/simple_view/view.sql",
-}
-
 
 class Errors(Enum):
     """DryRun errors that require special handling."""
@@ -299,12 +45,6 @@ class Errors(Enum):
 class DryRun:
     """Dry run SQL files."""
 
-    # todo: support different dry run endpoints for different projects
-    DRY_RUN_URL = (
-        "https://us-central1-moz-fx-data-shared-prod.cloudfunctions.net/"
-        "bigquery-etl-dryrun"
-    )
-
     def __init__(
         self,
         sqlfile,
@@ -321,14 +61,43 @@ class DryRun:
         self.use_cloud_function = use_cloud_function
         self.client = client if use_cloud_function or client else bigquery.Client()
         self.respect_skip = respect_skip
+        self.dry_run_url = ConfigLoader.get("dry_run", "function")
         try:
             self.metadata = Metadata.of_query_file(self.sqlfile)
         except FileNotFoundError:
             self.metadata = None
 
+    @staticmethod
+    def skipped_files() -> Set[str]:
+        """Return files skipped by dry run."""
+        skip_files = {
+            file
+            for skip in ConfigLoader.get("dry_run", "skip", fallback=[])
+            for file in glob.glob(
+                skip,
+                recursive=True,
+            )
+        }
+
+        # update skip list to include renamed queries in stage.
+        test_project = ConfigLoader.get("dry_run", "test_project", fallback="")
+        file_pattern_re = re.compile(r"sql/([^\/]+)/([^/]+)(/?.*|$)")
+        skip_files.update(
+            [
+                file
+                for skip in ConfigLoader.get("dry_run", "skip", fallback=[])
+                for file in glob.glob(
+                    file_pattern_re.sub(f"sql/{test_project}/\\2*\\3", skip),
+                    recursive=True,
+                )
+            ]
+        )
+
+        return skip_files
+
     def skip(self):
         """Determine if dry run should be skipped."""
-        return self.respect_skip and self.sqlfile in SKIP
+        return self.respect_skip and self.sqlfile in self.skipped_files()
 
     def get_sql(self):
         """Get SQL content."""
@@ -385,7 +154,7 @@ class DryRun:
             if self.use_cloud_function:
                 r = urlopen(
                     Request(
-                        self.DRY_RUN_URL,
+                        self.dry_run_url,
                         headers={"Content-Type": "application/json"},
                         data=json.dumps(
                             {
@@ -683,21 +452,3 @@ def find_next_word(target, source):
         if w == target:
             # get the next word, and remove quotations from column name
             return split[i + 1].replace("'", "")
-
-
-def add_test_project_to_skip(sql_dir="sql", project=TEST_PROJECT):
-    """Update skip list to include renamed queries in stage."""
-    SKIP.update(
-        [
-            p
-            for f in [Path(s) for s in SKIP]
-            for p in glob.glob(
-                f"sql/{TEST_PROJECT}/{f.parent.parent.name}*/{f.parent.name}/{f.name}",
-                recursive=True,
-            )
-        ]
-    )
-
-
-# detect test files on startup
-add_test_project_to_skip()

bqetl_project.yaml

@@ -0,0 +1,215 @@
+dry_run:
+  function: https://us-central1-moz-fx-data-shared-prod.cloudfunctions.net/bigquery-etl-dryrun
+  test_project: bigquery-etl-integration-test
+  skip:
+  # Access Denied
+  - sql/moz-fx-data-shared-prod/account_ecosystem_derived/ecosystem_client_id_lookup_v1/query.sql
+  - sql/moz-fx-data-shared-prod/account_ecosystem_derived/desktop_clients_daily_v1/query.sql
+  - sql/moz-fx-data-shared-prod/account_ecosystem_restricted/ecosystem_client_id_deletion_v1/query.sql
+  - sql/moz-fx-data-shared-prod/account_ecosystem_derived/fxa_logging_users_daily_v1/query.sql
+  - sql/moz-fx-data-shared-prod/activity_stream/impression_stats_flat/view.sql
+  - sql/moz-fx-data-shared-prod/activity_stream/tile_id_types/view.sql
+  - sql/moz-fx-data-shared-prod/monitoring_derived/deletion_request_volume_v1/query.sql
+  - sql/moz-fx-data-shared-prod/monitoring_derived/schema_error_counts_v1/view.sql
+  - sql/moz-fx-data-shared-prod/monitoring_derived/schema_error_counts_v2/query.sql
+  - sql/moz-fx-data-shared-prod/monitoring_derived/suggest_impression_rate_v1/init.sql
+  - sql/moz-fx-data-shared-prod/monitoring_derived/suggest_impression_rate_v1/query.sql
+  - sql/moz-fx-data-shared-prod/monitoring/schema_error_counts_v1/view.sql
+  - sql/moz-fx-data-shared-prod/monitoring_derived/structured_error_counts_v1/view.sql
+  - sql/moz-fx-data-shared-prod/monitoring/structured_error_counts/view.sql
+  - sql/moz-fx-data-shared-prod/monitoring_derived/telemetry_missing_columns_v1/view.sql
+  - sql/moz-fx-data-shared-prod/monitoring/telemetry_missing_columns_v1/view.sql
+  - sql/moz-fx-data-shared-prod/monitoring_derived/telemetry_missing_columns_v2/view.sql
+  - sql/moz-fx-data-shared-prod/monitoring/telemetry_missing_columns_v2/view.sql
+  - sql/moz-fx-data-shared-prod/monitoring*/suggest*_rate*_live*/*.sql
+  - sql/moz-fx-data-shared-prod/monitoring*/topsites*_rate*_live*/*.sql
+  - sql/moz-fx-data-shared-prod/pocket/pocket_reach_mau/view.sql
+  - sql/moz-fx-data-shared-prod/telemetry/buildhub2/view.sql
+  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_content_events_v1/query.sql
+  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_auth_bounce_events_v1/query.sql
+  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_auth_events_v1/query.sql
+  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_delete_events_v1/init.sql
+  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_delete_events_v1/query.sql
+  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_oauth_events_v1/query.sql
+  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_auth_events_v1/query.sql
+  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_content_events_v1/query.sql
+  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_log_device_command_events_v1/query.sql
+  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_first_seen_v1/query.sql
+  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_users_services_last_seen_v1/query.sql
+  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_amplitude_export_v1/query.sql
+  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_amplitude_user_ids_v1/query.sql
+  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_amplitude_user_ids_v1/init.sql
+  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/fxa_stdout_events_v1/query.sql
+  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/nonprod_fxa_auth_events_v1/query.sql
+  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/nonprod_fxa_content_events_v1/query.sql
+  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/nonprod_fxa_stdout_events_v1/query.sql
+  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/nonprod_fxa_server_events_v1/query.sql
+  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/docker_fxa_admin_server_sanitized_v1/init.sql
+  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/docker_fxa_admin_server_sanitized_v1/query.sql
+  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/docker_fxa_customs_sanitized_v1/init.sql
+  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/docker_fxa_customs_sanitized_v1/query.sql
+  - sql/moz-fx-data-shared-prod/fivetran_costs_derived/destinations_v1/query.sql
+  - sql/moz-fx-data-shared-prod/fivetran_costs_derived/incremental_mar_v1/query.sql
+  - sql/moz-fx-data-shared-prod/fivetran_costs_derived/monthly_costs_v1/query.sql
+  - sql/**/apple_ads_external*/**/query.sql
+  - sql/moz-fx-data-shared-prod/regrets_reporter/regrets_reporter_update/view.sql
+  - sql/moz-fx-data-shared-prod/revenue_derived/client_ltv_v1/query.sql
+  - sql/moz-fx-data-shared-prod/monitoring/payload_bytes_decoded_all/view.sql
+  - sql/moz-fx-data-shared-prod/monitoring/payload_bytes_decoded_structured/view.sql
+  - sql/moz-fx-data-shared-prod/monitoring/payload_bytes_decoded_stub_installer/view.sql
+  - sql/moz-fx-data-shared-prod/monitoring/payload_bytes_decoded_telemetry/view.sql
+  - sql/moz-fx-data-shared-prod/monitoring/payload_bytes_error_structured/view.sql
+  - sql/moz-fx-data-shared-prod/monitoring/payload_bytes_error_all/view.sql
+  - sql/moz-fx-data-shared-prod/monitoring_derived/shredder_progress/view.sql
+  - sql/moz-fx-data-shared-prod/monitoring/shredder_progress/view.sql
+  - sql/moz-fx-data-shared-prod/monitoring_derived/telemetry_distinct_docids_v1/query.sql
+  - sql/moz-fx-data-shared-prod/revenue_derived/client_ltv_normalized/query.sql
+  - sql/moz-fx-data-shared-prod/revenue_derived/client_ltv_normalized_v1/query.sql
+  - sql/moz-fx-data-shared-prod/stripe_derived/**/*.sql,
+  - sql/moz-fx-data-shared-prod/stripe_external/**/*.sql,
+  - sql/moz-fx-cjms-nonprod-9a36/cjms_bigquery/flows_live/view.sql
+  - sql/moz-fx-cjms-nonprod-9a36/cjms_bigquery/flows_v1/query.sql
+  - sql/moz-fx-cjms-nonprod-9a36/cjms_bigquery/refunds_v1/query.sql
+  - sql/moz-fx-cjms-nonprod-9a36/cjms_bigquery/subscriptions_v1/query.sql
+  - sql/moz-fx-cjms-prod-f3c7/cjms_bigquery/flows_v1/query.sql
+  - sql/moz-fx-cjms-prod-f3c7/cjms_bigquery/refunds_v1/query.sql
+  - sql/moz-fx-cjms-prod-f3c7/cjms_bigquery/subscriptions_v1/query.sql
+  - sql/moz-fx-data-shared-prod/subscription_platform_derived/apple_subscriptions_v1/query.sql
+  - sql/moz-fx-data-shared-prod/subscription_platform_derived/google_subscriptions_v1/query.sql
+  - sql/moz-fx-data-shared-prod/subscription_platform_derived/nonprod_apple_subscriptions_v1/query.sql
+  - sql/moz-fx-data-shared-prod/subscription_platform_derived/nonprod_google_subscriptions_v1/query.sql
+  - sql/moz-fx-data-shared-prod/subscription_platform_derived/nonprod_stripe_subscriptions_history_v1/query.sql
+  - sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_subscriptions_revised_changelog_v1/query.sql
+  - sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_subscriptions_changelog_v1/query.sql
+  - sql/moz-fx-data-shared-prod/subscription_platform_derived/stripe_subscriptions_history_v1/query.sql
+  - sql/moz-fx-data-shared-prod/stripe/itemized_payout_reconciliation/view.sql
+  - sql/moz-fx-data-shared-prod/mozilla_vpn_derived/add_device_events_v1/query.sql
+  - sql/moz-fx-data-shared-prod/mozilla_vpn_derived/devices_v1/query.sql
+  - sql/moz-fx-data-shared-prod/mozilla_vpn_derived/fxa_attribution_v1/query.sql
+  - sql/moz-fx-data-shared-prod/mozilla_vpn_derived/funnel_product_page_to_subscribed_v1/query.sql
+  - sql/moz-fx-data-shared-prod/mozilla_vpn_derived/guardian_apple_events_v1/query.sql
+  - sql/moz-fx-data-shared-prod/mozilla_vpn_derived/login_flows_v1/query.sql
+  - sql/moz-fx-data-shared-prod/mozilla_vpn_derived/protected_v1/query.sql
+  - sql/moz-fx-data-shared-prod/mozilla_vpn_derived/site_metrics_empty_check_v1/query.sql
+  - sql/moz-fx-data-shared-prod/mozilla_vpn_derived/site_metrics_summary_v1/query.sql
+  - sql/moz-fx-data-shared-prod/mozilla_vpn_derived/subscriptions_v1/query.sql
+  - sql/moz-fx-data-shared-prod/mozilla_vpn_derived/users_v1/query.sql
+  - sql/moz-fx-data-shared-prod/mozilla_vpn_external/devices_v1/query.sql
+  - sql/moz-fx-data-shared-prod/mozilla_vpn_external/subscriptions_v1/query.sql
+  - sql/moz-fx-data-shared-prod/mozilla_vpn_external/users_v1/query.sql
+  - sql/moz-fx-data-shared-prod/monitoring_derived/telemetry_missing_columns_v3/query.sql
+  - sql/moz-fx-data-experiments/monitoring/query_cost_v1/query.sql
+  - sql/moz-fx-data-shared-prod/mozilla_vpn_external/subscriptions_v1/init.sql
+  - sql/moz-fx-data-shared-prod/mozilla_vpn_external/users_v1/init.sql
+  - sql/moz-fx-data-shared-prod/mozilla_vpn_derived/protected_v1/init.sql
+  - sql/moz-fx-data-shared-prod/mozilla_vpn_derived/add_device_events_v1/init.sql
+  - sql/moz-fx-data-shared-prod/mozilla_vpn_external/devices_v1/init.sql
+  - sql/moz-fx-data-shared-prod/fenix_derived/google_ads_campaign_cost_breakdowns_v1/query.sql
+  - sql/moz-fx-data-shared-prod/search_terms*/**/*.sql
+  - sql/moz-fx-data-bq-performance/release_criteria/dashboard_health_v1/query.sql
+  - sql/moz-fx-data-bq-performance/release_criteria/rc_flattened_test_data_v1/query.sql
+  - sql/moz-fx-data-bq-performance/release_criteria/release_criteria_summary_v1/query.sql
+  - sql/moz-fx-data-bq-performance/release_criteria/stale_tests_v1/query.sql
+  - sql/moz-fx-data-bq-performance/release_criteria/release_criteria_v1/query.sql
+  - sql/moz-fx-data-shared-prod/contextual_services/**/*.sql
+  - sql/moz-fx-data-shared-prod/contextual_services_derived/**/*.sql
+  - sql/moz-fx-data-shared-prod/**/topsites_impression/view.sql
+  - sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql
+  - sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/init.sql
+  - sql/moz-fx-data-shared-prod/contextual_services_derived/adm_forecasting_v1/query.sql
+  - sql/moz-fx-data-shared-prod/regrets_reporter/regrets_reporter_summary/view.sql
+  - sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql
+  - sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_tiles_v2/query.sql
+  - sql/moz-fx-data-shared-prod/regrets_reporter_derived/regrets_reporter_summary_v1/*.sql
+  - sql/moz-fx-data-shared-prod/mlhackweek_search/**/*.sql
+  - sql/moz-fx-data-shared-prod/regrets_reporter_ucs/**/*.sql
+  - sql/moz-fx-data-shared-prod/telemetry/xfocsp_error_report/view.sql
+  - sql/moz-fx-data-shared-prod/telemetry/regrets_reporter_update/view.sql
+  - sql/moz-fx-data-marketing-prod/acoustic/**/*.sql
+  - sql/moz-fx-data-marketing-prod/adjust_derived/**/*.sql
+  - sql/moz-fx-data-shared-prod/monitoring_derived/airflow_*/*.sql
+  - sql/moz-fx-data-shared-prod/google_ads_derived/**/*.sql
+  - sql/moz-fx-data-shared-prod/**/client_deduplication*/*.sql
+  - sql/moz-fx-data-marketing-prod/ga_derived/downloads_with_attribution_v1/query.sql
+  - sql/moz-fx-data-marketing-prod/ga_derived/downloads_with_attribution_v2/query.sql
+  - sql/moz-fx-data-shared-prod/fenix_external/installs_by_country_v1/query.sql
+  - sql/moz-fx-data-shared-prod/fenix/installs_by_country/view.sql
+  - sql/moz-fx-data-shared-prod/firefox_desktop/top_sites/view.sql
+  - sql/moz-fx-data-shared-prod/firefox_desktop/quick_suggest/view.sql
+  # Materialized views
+  - sql/moz-fx-data-shared-prod/telemetry_derived/experiment_search_events_live_v1/init.sql
+  - sql/moz-fx-data-shared-prod/telemetry_derived/experiment_events_live_v1/init.sql
+  - sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/experiment_search_events_live_v1/init.sql
+  - sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/experiment_events_live_v1/init.sql
+  - sql/moz-fx-data-shared-prod/org_mozilla_firefox_derived/experiment_search_events_live_v1/init.sql
+  - sql/moz-fx-data-shared-prod/org_mozilla_firefox_derived/experiment_events_live_v1/init.sql
+  - sql/moz-fx-data-shared-prod/org_mozilla_firefox_beta_derived/experiment_search_events_live_v1/init.sql
+  - sql/moz-fx-data-shared-prod/org_mozilla_firefox_beta_derived/experiment_events_live_v1/init.sql
+  - sql/moz-fx-data-shared-prod/telemetry_derived/experiment_enrollment_cumulative_population_estimate_v1/view.sql
+  - sql/moz-fx-data-shared-prod/telemetry/experiment_enrollment_cumulative_population_estimate/view.sql
+  # Already exists (and lacks an "OR REPLACE" clause)
+  - sql/moz-fx-data-shared-prod/org_mozilla_firefox_derived/clients_first_seen_v1/init.sql
+  - sql/moz-fx-data-shared-prod/org_mozilla_firefox_derived/clients_last_seen_v1/init.sql
+  - sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/clients_last_seen_v1/init.sql
+  - sql/moz-fx-data-shared-prod/org_mozilla_vrbrowser_derived/clients_last_seen_v1/init.sql
+  - sql/moz-fx-data-shared-prod/telemetry_derived/core_clients_last_seen_v1/init.sql
+  - sql/moz-fx-data-shared-prod/telemetry/fxa_users_last_seen_raw_v1/init.sql
+  - sql/moz-fx-data-shared-prod/telemetry_derived/core_clients_first_seen_v1/init.sql
+  - sql/moz-fx-data-shared-prod/telemetry_derived/fxa_users_services_last_seen_v1/init.sql
+  - sql/moz-fx-data-shared-prod/messaging_system_derived/cfr_users_last_seen_v1/init.sql
+  - sql/moz-fx-data-shared-prod/messaging_system_derived/onboarding_users_last_seen_v1/init.sql
+  - sql/moz-fx-data-shared-prod/messaging_system_derived/snippets_users_last_seen_v1/init.sql
+  - sql/moz-fx-data-shared-prod/messaging_system_derived/whats_new_panel_users_last_seen_v1/init.sql
+  # Reference table not found
+  - sql/moz-fx-data-shared-prod/monitoring_derived/structured_detailed_error_counts_v1/view.sql
+  - sql/moz-fx-data-shared-prod/monitoring/structured_detailed_error_counts/view.sql
+  - sql/moz-fx-data-shared-prod/org_mozilla_firefox_derived/migrated_clients_v1/query.sql
+  - sql/moz-fx-data-shared-prod/org_mozilla_firefox_derived/incline_executive_v1/query.sql
+  - sql/moz-fx-data-shared-prod/org_mozilla_firefox/migrated_clients/view.sql
+  - sql/moz-fx-data-shared-prod/telemetry/sponsored_tiles_ad_request_fill/view.sql
+  # No matching signature for function IF
+  - sql/moz-fx-data-shared-prod/static/fxa_amplitude_export_users_last_seen/query.sql
+  # Duplicate UDF
+  - sql/moz-fx-data-shared-prod/static/fxa_amplitude_export_users_daily/query.sql
+  # Syntax error
+  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_last_seen_v1/init.sql
+  # HTTP Error 408: Request Time-out
+  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_first_seen_v1/query.sql
+  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_last_seen_v1/query.sql
+  - sql/moz-fx-data-shared-prod/telemetry_derived/latest_versions/query.sql
+  - sql/moz-fx-data-shared-prod/telemetry_derived/italy_covid19_outage_v1/query.sql
+  - sql/moz-fx-data-shared-prod/telemetry_derived/main_nightly_v1/init.sql
+  - sql/moz-fx-data-shared-prod/telemetry_derived/main_nightly_v1/query.sql
+  - sql/moz-fx-data-shared-prod/telemetry_derived/main_1pct_v1/init.sql
+  - sql/moz-fx-data-shared-prod/telemetry_derived/main_1pct_v1/query.sql
+  # Query parameter not found
+  - sql/moz-fx-data-shared-prod/telemetry_derived/experiments_v1/query.sql
+  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_scalar_aggregates_v1/query.sql
+  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_keyed_scalar_aggregates_v1/query.sql
+  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_keyed_boolean_aggregates_v1/query.sql
+  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_histogram_aggregates_content_v1/query.sql
+  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_histogram_aggregates_gpu_v1/query.sql
+  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_histogram_aggregates_parent_v1/query.sql
+  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_keyed_histogram_aggregates_v1/query.sql
+  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_histogram_aggregates_v1/query.sql
+  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_histogram_aggregates_v2/query.sql
+  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_histogram_bucket_counts_v1/query.sql
+  - sql/moz-fx-data-shared-prod/telemetry_derived/glam_client_probe_counts_extract_v1/query.sql
+  - sql/moz-fx-data-shared-prod/telemetry_derived/glam_sample_counts_v1/query.sql
+  - sql/moz-fx-data-shared-prod/telemetry_derived/glam_user_counts_v1/query.sql
+  - sql/moz-fx-data-shared-prod/telemetry_derived/scalar_percentiles_v1/query.sql
+  - sql/moz-fx-data-shared-prod/telemetry_derived/clients_scalar_probe_counts_v1/query.sql
+  - sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_etl_sql_run_check_v1/query.sql
+  # Dataset sql/glam-fenix-dev:glam_etl was not found
+  - sql/glam-fenix-dev/glam_etl/**/*.sql
+  # Query templates
+  - sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/fenix_metrics.template.sql
+  - sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/mobile_search_clients_daily.template.sql
+  # Temporary table does not exist
+  # TODO: remove this in a follow up PR
+  - sql/moz-fx-data-shared-prod/*/baseline_clients_first_seen_v1/*.sql
+  # Query too complex
+  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/event_types_history_v1/query.sql
+  - sql/moz-fx-data-shared-prod/firefox_accounts_derived/event_types_history_v1/init.sql
+  # Tests
+  - sql/moz-fx-data-test-project/test/simple_view/view.sql

tests/data/bqetl_project.yaml

@@ -0,0 +1,7 @@
+dry_run:
+  function: https://us-central1-moz-fx-data-shared-prod.cloudfunctions.net/bigquery-etl-dryrun
+  test_project: bigquery-etl-integration-test
+  skip:
+  # Access Denied
+  - sql/moz-fx-data-shared-prod/test_derived/some_query_v1/query.sql
+  - sql/moz-fx-data-shared-prod/test_derived/another_query_v1/query.sql

tests/test_config.py

@@ -0,0 +1,34 @@
+from pathlib import Path
+
+from bigquery_etl.config import ConfigLoader
+
+TEST_DIR = Path(__file__).parent
+
+
+class TestConfig:
+    def test_config_loader_set_project(self):
+        ConfigLoader.set_project_dir(TEST_DIR / "data")
+        assert ConfigLoader.project_dir == TEST_DIR / "data"
+
+    def test_config_loader_get(self):
+        ConfigLoader.set_project_dir(TEST_DIR / "data")
+
+        assert "function" in ConfigLoader.get("dry_run")
+        assert "test_project" in ConfigLoader.get("dry_run")
+        assert "skip" in ConfigLoader.get("dry_run")
+
+        assert (
+            ConfigLoader.get("dry_run", "test_project")
+            == "bigquery-etl-integration-test"
+        )
+        assert len(ConfigLoader.get("dry_run", "skip")) == 2
+
+        assert "dry_run" in ConfigLoader.get()
+
+    def test_config_loader_get_non_existing(self):
+        ConfigLoader.set_project_dir(TEST_DIR / "data")
+
+        assert ConfigLoader.get("non_existing") is None
+        assert ConfigLoader.get("dry_run", "non_existing") is None
+        assert ConfigLoader.get("non_existing", fallback=[]) == []
+        assert ConfigLoader.get("dry_run", "foo", fallback=123) == 123