Move doc validation and generation commands to CLI

.circleci/config.yml

@@ -311,7 +311,7 @@ jobs:
           name: Build and deploy docs
           command: |
             rm -r sql/ && cp -r /tmp/workspace/generated-sql/sql sql/
-            PATH="venv/bin:$PATH" script/generate_docs \
+            PATH="venv/bin:$PATH" script/bqetl docs generate \
                --output_dir=generated_docs/
             cd generated_docs/
             PATH="../venv/bin:$PATH" mkdocs gh-deploy \

bigquery_etl/cli/__init__.py

@@ -16,6 +16,7 @@ from ..cli.query import query
 from ..cli.routine import mozfun, routine
 from ..cli.view import view
 from ..dependency import dependency
+from ..docs import docs_
 from ..glam.cli import glam
 from ..static import static_
 from ..stripe import stripe_
@@ -39,6 +40,7 @@ def cli(prog_name=None):
         "alchemer": alchemer_,
         "apple": apple,
         "static": static_,
+        "docs": docs_,
     }
 
     @click.group(commands=commands)

bigquery_etl/cli/routine.py

@@ -15,7 +15,7 @@ import yaml
 
 from ..cli.format import format
 from ..cli.utils import is_authenticated, is_valid_dir, is_valid_project
-from ..docs import validate_docs
+from ..docs import validate as validate_docs
 from ..format_sql.formatter import reformat
 from ..routine import publish_routines
 from ..routine.parse_routine import PROCEDURE_FILE, UDF_FILE
@@ -370,7 +370,7 @@ def validate(ctx, name, sql_dir, project_id, docs_only):
 
     routine_files = _routines_matching_name_pattern(name, sql_dir, project_id)
 
-    validate_docs.validate(project_dirs(project_id))
+    ctx.invoke(validate_docs, project_dirs=project_dirs(project_id))
 
     if not docs_only:
         for routine_file in routine_files:

bigquery_etl/docs/__init__.py

@@ -0,0 +1,132 @@
+"""Docs."""
+
+import logging
+import os
+import shutil
+import sys
+import tempfile
+from pathlib import Path
+
+import click
+
+from bigquery_etl.docs.derived_datasets.generate_derived_dataset_docs import (
+    generate_derived_dataset_docs,
+)
+from bigquery_etl.dryrun import DryRun
+
+DEFAULT_PROJECTS_DIRS = ["sql/mozfun/", "sql/moz-fx-data-shared-prod/"]
+DOCS_DIR = "docs/"
+EXAMPLE_DIR = "examples"
+INDEX_MD = "index.md"
+
+
+@click.group("docs", help="Commands for generated documentation.")
+def docs_():
+    """Create the CLI group for doc commands."""
+    pass
+
+
+project_dirs_option = click.option(
+    "--project_dirs",
+    "--project-dirs",
+    help="Directories of projects documentation is generated for.",
+    multiple=True,
+    default=DEFAULT_PROJECTS_DIRS,
+)
+
+log_level_option = click.option(
+    "--log-level",
+    "--log_level",
+    help="Log level.",
+    default=logging.getLevelName(logging.INFO),
+    type=str.upper,
+)
+
+
+@docs_.command("generate", help="Generate the project docs.")
+@project_dirs_option
+@click.option(
+    "--docs_dir",
+    "--docs-dir",
+    default=DOCS_DIR,
+    help="Directory containing static documentation.",
+)
+@click.option(
+    "--output_dir",
+    "--output-dir",
+    required=True,
+    help="Generated documentation is written to this output directory.",
+)
+@log_level_option
+def generate(project_dirs, docs_dir, output_dir, log_level):
+    """Generate documentation for project."""
+    from bigquery_etl.docs.bqetl.generate_bqetl_docs import generate_bqetl_docs
+    from bigquery_etl.docs.mozfun.generate_mozfun_docs import generate_mozfun_docs
+
+    out_dir = os.path.join(output_dir, "docs")
+
+    # To customize Mkdocs, we need to extend the theme with an `overrides` folder
+    # https://squidfunk.github.io/mkdocs-material/customization/#overriding-partials
+    override_dir = os.path.join(output_dir, "overrides")
+
+    if os.path.exists(out_dir) and os.path.exists(override_dir):
+        shutil.rmtree(out_dir)
+        shutil.rmtree(override_dir)
+
+    # copy assets from /docs and /overrides folders to output folder
+    shutil.copytree(docs_dir, out_dir)
+    shutil.copytree("bigquery_etl/docs/overrides", override_dir)
+
+    # move mkdocs.yml out of docs/
+    mkdocs_path = os.path.join(output_dir, "mkdocs.yml")
+    shutil.move(os.path.join(out_dir, "mkdocs.yml"), mkdocs_path)
+
+    # generate bqetl command docs
+    generate_bqetl_docs(Path(out_dir) / "bqetl.md")
+
+    # move files to docs/
+    for project_dir in project_dirs:
+        if not os.path.isdir(project_dir):
+            continue
+
+        if "mozfun" in project_dir:
+            generate_mozfun_docs(out_dir, project_dir)
+        else:
+            generate_derived_dataset_docs(out_dir, project_dir)
+
+
+@docs_.command("validate", help="Validate the project docs.")
+@project_dirs_option
+@log_level_option
+def validate(project_dirs, log_level):
+    """Validate UDF docs."""
+    from bigquery_etl.routine.parse_routine import read_routine_dir, sub_local_routines
+
+    is_valid = True
+
+    for project_dir in project_dirs:
+        if os.path.isdir(project_dir):
+            parsed_routines = read_routine_dir(project_dir)
+
+            for root, dirs, files in os.walk(project_dir):
+                if os.path.basename(root) == EXAMPLE_DIR:
+                    sql_files = (f for f in files if os.path.splitext(f)[1] == ".sql")
+                    for file in sql_files:
+                        dry_run_sql = sub_local_routines(
+                            (Path(root) / file).read_text(),
+                            project_dir,
+                            parsed_routines,
+                        )
+
+                        # store sql in temporary file for dry_run
+                        tmp_dir = Path(tempfile.mkdtemp()) / Path(root)
+                        tmp_dir.mkdir(parents=True, exist_ok=True)
+                        tmp_example_file = tmp_dir / file
+                        tmp_example_file.write_text(dry_run_sql)
+
+                        if not DryRun(str(tmp_example_file)).is_valid():
+                            is_valid = False
+
+    if not is_valid:
+        print("Invalid examples.")
+        sys.exit(1)

bigquery_etl/docs/bqetl/generate_bqetl_docs.py

@@ -1,9 +1,10 @@
 """Generate docs for bqetl commands."""
 
+import os
+from pathlib import Path
+
 import click
 from jinja2 import Environment, FileSystemLoader
-import os
-from pathlib import Path
 
 from bigquery_etl.cli.dag import dag
 from bigquery_etl.cli.dryrun import dryrun
@@ -12,7 +13,6 @@ from bigquery_etl.cli.query import query
 from bigquery_etl.cli.routine import mozfun, routine
 from bigquery_etl.dependency import dependency
 
-
 # commands to document
 COMMANDS = {
     "query": query,

bigquery_etl/docs/generate_docs.py

@@ -1,78 +0,0 @@
-"""Generates documentations for provided projects."""
-
-import os
-import shutil
-from argparse import ArgumentParser
-from pathlib import Path
-
-from bigquery_etl.docs.bqetl.generate_bqetl_docs import generate_bqetl_docs
-from bigquery_etl.docs.derived_datasets.generate_derived_dataset_docs import (
-    generate_derived_dataset_docs,
-)
-from bigquery_etl.docs.mozfun.generate_mozfun_docs import generate_mozfun_docs
-from bigquery_etl.util import standard_args
-
-DEFAULT_PROJECTS_DIRS = ["sql/mozfun/", "sql/moz-fx-data-shared-prod/"]
-DOCS_DIR = "docs/"
-INDEX_MD = "index.md"
-
-parser = ArgumentParser(description=__doc__)
-parser.add_argument(
-    "--project_dirs",
-    "--project-dirs",
-    help="Directories of projects documentation is generated for.",
-    nargs="+",
-    default=DEFAULT_PROJECTS_DIRS,
-)
-parser.add_argument(
-    "--docs_dir",
-    "--docs-dir",
-    default=DOCS_DIR,
-    help="Directory containing static documentation.",
-)
-parser.add_argument(
-    "--output_dir",
-    "--output-dir",
-    required=True,
-    help="Generated documentation is written to this output directory.",
-)
-standard_args.add_log_level(parser)
-
-
-def main():
-    """Generate documentation for project."""
-    args = parser.parse_args()
-    out_dir = os.path.join(args.output_dir, "docs")
-
-    # To customize Mkdocs, we need to extend the theme with an `overrides` folder
-    # https://squidfunk.github.io/mkdocs-material/customization/#overriding-partials
-    override_dir = os.path.join(args.output_dir, "overrides")
-
-    if os.path.exists(out_dir) and os.path.exists(override_dir):
-        shutil.rmtree(out_dir)
-        shutil.rmtree(override_dir)
-
-    # copy assets from /docs and /overrides folders to output folder
-    shutil.copytree(args.docs_dir, out_dir)
-    shutil.copytree("bigquery_etl/docs/overrides", override_dir)
-
-    # move mkdocs.yml out of docs/
-    mkdocs_path = os.path.join(args.output_dir, "mkdocs.yml")
-    shutil.move(os.path.join(out_dir, "mkdocs.yml"), mkdocs_path)
-
-    # generate bqetl command docs
-    generate_bqetl_docs(Path(out_dir) / "bqetl.md")
-
-    # move files to docs/
-    for project_dir in args.project_dirs:
-        if not os.path.isdir(project_dir):
-            continue
-
-        if "mozfun" in project_dir:
-            generate_mozfun_docs(out_dir, project_dir)
-        else:
-            generate_derived_dataset_docs(out_dir, project_dir)
-
-
-if __name__ == "__main__":
-    main()

bigquery_etl/docs/validate_docs.py

@@ -1,69 +0,0 @@
-"""Validates SQL examples in documentations."""
-
-import os
-import sys
-import tempfile
-from argparse import ArgumentParser
-from pathlib import Path
-
-from bigquery_etl.dryrun import DryRun
-from bigquery_etl.routine.parse_routine import read_routine_dir, sub_local_routines
-from bigquery_etl.util import standard_args
-
-DEFAULT_PROJECTS_DIRS = ["sql/mozfun"]
-EXAMPLE_DIR = "examples"
-UDF_FILE = "udf.sql"
-UDF_CHAR = "[a-zA-z0-9_]"
-MOZFUN_UDF_RE = rf"mozfun.({UDF_CHAR}+\.{UDF_CHAR}+)"
-
-parser = ArgumentParser(description=__doc__)
-parser.add_argument(
-    "--project_dirs",
-    "--project-dirs",
-    help="Directories of projects documentation is validated for.",
-    nargs="+",
-    default=DEFAULT_PROJECTS_DIRS,
-)
-standard_args.add_log_level(parser)
-
-
-def validate(project_dirs):
-    """Validate UDF docs."""
-    is_valid = True
-
-    for project_dir in project_dirs:
-        if os.path.isdir(project_dir):
-            parsed_routines = read_routine_dir(project_dir)
-
-            for root, dirs, files in os.walk(project_dir):
-                if os.path.basename(root) == EXAMPLE_DIR:
-                    sql_files = (f for f in files if os.path.splitext(f)[1] == ".sql")
-                    for file in sql_files:
-                        dry_run_sql = sub_local_routines(
-                            (Path(root) / file).read_text(),
-                            project_dir,
-                            parsed_routines,
-                        )
-
-                        # store sql in temporary file for dry_run
-                        tmp_dir = Path(tempfile.mkdtemp()) / Path(root)
-                        tmp_dir.mkdir(parents=True, exist_ok=True)
-                        tmp_example_file = tmp_dir / file
-                        tmp_example_file.write_text(dry_run_sql)
-
-                        if not DryRun(str(tmp_example_file)).is_valid():
-                            is_valid = False
-
-    if not is_valid:
-        print("Invalid examples.")
-        sys.exit(1)
-
-
-def main():
-    """Validate SQL examples."""
-    args = parser.parse_args()
-    validate(args.project_dirs)
-
-
-if __name__ == "__main__":
-    main()

docs/bqetl.md

@@ -44,4 +44,4 @@ See help for any command:
 $ ./bqetl [command] --help
 ```
 
-<!--- Documentation for comments is generated via bigquery_etl/docs/generate_docs -->
+<!--- Documentation for comments is generated via ./bqetl docs generate -->

docs/cookbooks/common_workflows.md

@@ -277,6 +277,6 @@ executed.
 The [repository documentation](https://mozilla.github.io/bigquery-etl/) is built using [MkDocs](https://www.mkdocs.org/).
 To generate and check the docs locally:
 
-1. Run `script/generate_docs --output_dir generated_docs`
+1. Run `./bqetl docs generate --output_dir generated_docs`
 1. Navigate to the `generated_docs` directory
 1. Run `mkdocs serve` to start a local `mkdocs` server.

netlify.toml

@@ -12,7 +12,7 @@ command = """
     cp -r sql/ sql-output/sql
     ./script/generate_sql --sql-dir sql-output/sql --target-project moz-fx-data-shared-prod
     rm -r sql/ && mv sql-output/sql sql/
-    ./script/generate_docs --output_dir=generated_docs/
+    ./script/bqetl docs generate --output_dir=generated_docs/
     cd generated_docs && mkdocs build
 """
 environment = { PYTHON_VERSION = "3.8" }

script/generate_docs

@@ -1,8 +0,0 @@
-#!/bin/sh
-
-# Generate documentation for projects.
-
-
-cd "$(dirname "$0")/.."
-
-exec python3 -m bigquery_etl.docs.generate_docs "$@"