Add support for --sql-dir option (#3708)

.circleci/config.yml

@@ -346,7 +346,8 @@ jobs:
               --target-project moz-fx-data-shared-prod
             PATH="venv/bin:$PATH" script/bqetl query render \
               --sql-dir /tmp/workspace/generated-sql/sql/ \
-              --output-dir /tmp/workspace/generated-sql/sql/
+              --output-dir /tmp/workspace/generated-sql/sql/ \
+              /tmp/workspace/generated-sql/sql/
             PATH="venv/bin:$PATH" script/bqetl dependency record \
               --skip-existing \
               "/tmp/workspace/generated-sql/sql/"

bigquery_etl/cli/query.py

@@ -35,6 +35,7 @@ from ..cli.utils import (
 )
 from ..dependency import get_dependency_graph
 from ..dryrun import SKIP, DryRun
+from ..format_sql.format import SKIP as SKIP_FORMAT
 from ..format_sql.formatter import reformat
 from ..metadata import validate_metadata
 from ..metadata.parse_metadata import (
@@ -1230,10 +1231,19 @@ def render(name, sql_dir, output_dir):
     query_files = paths_matching_name_pattern(name, sql_dir, project_id=None)
     resolved_sql_dir = Path(sql_dir).resolve()
     for query_file in query_files:
-        rendered_sql = render_template(
-            query_file.name, template_folder=query_file.parent, templates_dir=""
+        rendered_sql = (
+            render_template(
+                query_file.name,
+                template_folder=query_file.parent,
+                templates_dir="",
+                format=False,
+            )
+            + "\n"
         )
 
+        if not any(s in str(query_file) for s in SKIP_FORMAT):
+            rendered_sql = reformat(rendered_sql, trailing_newline=True)
+
         if output_dir:
             output_file = output_dir / query_file.resolve().relative_to(
                 resolved_sql_dir

bigquery_etl/cli/utils.py

@@ -122,6 +122,7 @@ def paths_matching_name_pattern(pattern, sql_path, project_id, files=["*.sql"]):
 
 sql_dir_option = click.option(
     "--sql_dir",
+    "--sql-dir",
     help="Path to directory which contains queries.",
     type=click.Path(file_okay=False),
     default="sql",

sql/moz-fx-data-shared-prod/mozilla_vpn_external/devices_v1/query.sql

@@ -6,7 +6,7 @@ FROM
     -- The external_database_query argument in EXTERNAL_QUERY must be a literal string or query
     -- parameter, and cannot be generated at runtime using function calls like CONCAT or FORMAT,
     -- so the entire value must be provided as a STRING query parameter to handle specific dates:
-    -- "SELECT * FROM devices WHERE DATE(updated_at) = DATE '{{ds}}'"
+    -- {% raw %} "SELECT * FROM devices WHERE DATE(updated_at) = DATE '{{ds}}'" {% endraw %}
     @external_database_query
   ) AS _update
 FULL JOIN

sql/moz-fx-data-shared-prod/mozilla_vpn_external/subscriptions_v1/query.sql

@@ -6,7 +6,7 @@ FROM
     -- The external_database_query argument in EXTERNAL_QUERY must be a literal string or query
     -- parameter, and cannot be generated at runtime using function calls like CONCAT or FORMAT,
     -- so the entire value must be provided as a STRING query parameter to handle specific dates:
-    -- "SELECT * FROM subscriptions WHERE DATE(updated_at) = DATE '{{ds}}'"
+    -- {% raw %} "SELECT * FROM subscriptions WHERE DATE(updated_at) = DATE '{{ds}}'" {% endraw %}
     @external_database_query
   ) AS _update
 FULL JOIN

sql/moz-fx-data-shared-prod/mozilla_vpn_external/users_v1/query.sql

@@ -11,7 +11,7 @@ FROM
     -- The external_database_query argument in EXTERNAL_QUERY must be a literal string or query
     -- parameter, and cannot be generated at runtime using function calls like CONCAT or FORMAT,
     -- so the entire value must be provided as a STRING query parameter to handle specific dates:
-    -- "SELECT * FROM users WHERE DATE(updated_at) = DATE '{{ds}}'"
+    -- {% raw %} "SELECT * FROM users WHERE DATE(updated_at) = DATE '{{ds}}'" {% endraw %}
     @external_database_query
   ) AS _current
 FULL JOIN