Limit the projects scanned for routines (#6101)

.circleci/workflows.yml

@@ -149,6 +149,8 @@ jobs:
                   PATH="venv/bin:$PATH" script/entrypoint --black --flake8 \
                     --isort --mypy-ignore-missing-imports --pydocstyle \
                     -m "not (routine or sql or integration)" \
+                    -p no:bigquery_etl.pytest_plugin.routine \
+                    -p no:bigquery_etl.pytest_plugin.sql \
                     -n 8
       - unless:
           condition: *validate-bqetl
@@ -178,7 +180,7 @@ jobs:
             - run:
                 name: Run SQL tests
                 command: |
-                    PATH="venv/bin:$PATH" script/entrypoint -m sql -n 8
+                    PATH="venv/bin:$PATH" script/entrypoint -m sql -n 8 -p no:bigquery_etl.pytest_plugin.routine
             - &copy_debug_sql
               run:
                 name: Copy generated SQL to save for debugging
@@ -310,7 +312,9 @@ jobs:
                 name: PyTest Integration Test
                 # yamllint disable rule:line-length
                 command: |
-                  PATH="venv/bin:$PATH" script/entrypoint -m 'integration'
+                  PATH="venv/bin:$PATH" script/entrypoint -m 'integration' -n 8 \
+                  -p no:bigquery_etl.pytest_plugin.routine \
+                  -p no:bigquery_etl.pytest_plugin.sql
                 # workaround for job failing with `Too long with no output (exceeded 10m0s): context deadline exceeded` error
                 no_output_timeout: 30m
       - unless:

bigquery_etl/pytest_plugin/routine.py

@@ -7,6 +7,7 @@ import pytest
 from google.api_core.exceptions import BadRequest
 from google.cloud import bigquery
 
+from bigquery_etl.config import ConfigLoader
 from bigquery_etl.util.common import project_dirs
 
 from ..routine.parse_routine import (
@@ -26,7 +27,9 @@ def parsed_routines():
     if _parsed_routines is None:
         _parsed_routines = {
             routine.filepath: routine
-            for project in project_dirs()
+            for project in ConfigLoader.get(
+                "routine", "test_projects", fallback=project_dirs()
+            )
             for routine in parse_routines(project)
         }
 
@@ -52,6 +55,7 @@ class RoutineFile(pytest.File):
         """Collect."""
         self.add_marker("routine")
         self.routine = parsed_routines()[self.name]
+
         for i, query in enumerate(self.routine.tests_full_sql):
             yield RoutineTest.from_parent(
                 self, name=f"{self.routine.name}#{i+1}", query=query

bqetl_project.yaml

@@ -219,6 +219,10 @@ routine:
     - sql/moz-fx-data-shared-prod/udf/main_summary_scalars/udf.sql
   example_dir: examples
   project: mozfun
+  test_projects:
+  - sql/mozfun
+  - sql/moz-fx-data-shared-prod
+  - sql/bigquery-etl-integration-test
 
 view:
   validation: