Use jinja macros for recursive bqetl docs generation

bigquery_etl/cli/alchemer.py

@@ -14,11 +14,11 @@ Examples:
 \b
 # Import data from alchemer (surveygizmo) surveys into BigQuery.
 # The date range is inclusive of the start and end values.
-$ ./bqetl alchemer backfill --start-date=2021-01-01
-    --end-date=2021-02-01
-    --survey_id=xxxxxxxxxxx
-    --api_token=xxxxxxxxxxxxxx
-    --api_secret=xxxxxxxxxxxxxxx
+$ ./bqetl alchemer backfill --start-date=2021-01-01 \\
+    --end-date=2021-02-01 \\
+    --survey_id=xxxxxxxxxxx \\
+    --api_token=xxxxxxxxxxxxxx \\
+    --api_secret=xxxxxxxxxxxxxxx \\
     --destination_table=moz-fx-data-shared-prod.telemetry_derived.survey_gizmo_daily_attitudes
 """
 )

bigquery_etl/docs/bqetl/generate_bqetl_docs.py

@@ -30,7 +30,7 @@ TEMPLATE = FILE_PATH / "templates" / "commands.md"
 
 def extract_description_from_help_text(help_text):
     """Return the description from the command help text."""
-    return help_text.split("\n\n")[0]
+    return help_text.split("\n\n")[0].strip()
 
 
 def extract_examples_from_help_text(help_text):
@@ -41,6 +41,8 @@ def extract_examples_from_help_text(help_text):
     if len(help_text) > 1:
         examples = "\n\n".join(help_text[1:])
         examples = examples.replace("Examples:\n\n", "")
+        examples = examples.replace("\b", "")
+        examples = examples.strip()
 
     return examples
 
@@ -59,46 +61,29 @@ def extract_arguments_from_command(cmd):
     return [{"name": arg.name} for arg in cmd.params if isinstance(arg, click.Argument)]
 
 
+def parse_commands(cmd, path):
+    """Parse click commands and store in dict."""
+    commands = []
+    # full command path; reflect click group nesting
+    path = path + " " + cmd.name
+    if isinstance(cmd, click.Group):
+        commands = [parse_commands(c, path) for _, c in cmd.commands.items()]
+
+    return {
+        "name": cmd.name,
+        "commands": commands,
+        "examples": extract_examples_from_help_text(cmd.help),
+        "description": extract_description_from_help_text(cmd.help),
+        "options": extract_options_from_command(cmd),
+        "arguments": extract_arguments_from_command(cmd),
+        "path": path,
+    }
+
+
 def generate_bqetl_docs(out_file):
     """Generate documentation for bqetl CLI commands."""
     print("Generate bqetl command docs.")
-    command_groups = []
-
-    for command_group_name, command_group in COMMANDS.items():
-        commands = []
-        try:
-            for _, command in command_group.commands.items():
-                commands.append(
-                    {
-                        "name": command.name,
-                        "description": extract_description_from_help_text(command.help),
-                        "examples": extract_examples_from_help_text(command.help),
-                        "options": extract_options_from_command(command),
-                        "arguments": extract_arguments_from_command(command),
-                    }
-                )
-
-            command_groups.append(
-                {
-                    "name": command_group_name,
-                    "commands": commands,
-                    "description": command_group.help,
-                }
-            )
-        except Exception:
-            # command is not a group, but simply a click.Command
-            command_groups.append(
-                {
-                    "name": command_group_name,
-                    "commands": [],
-                    "examples": extract_examples_from_help_text(command_group.help),
-                    "description": extract_description_from_help_text(
-                        command_group.help
-                    ),
-                    "options": extract_options_from_command(command_group),
-                    "arguments": extract_arguments_from_command(command_group),
-                }
-            )
+    command_groups = [parse_commands(group, "") for _, group in COMMANDS.items()]
 
     # render md docs
     file_loader = FileSystemLoader(TEMPLATE.parent)
@@ -109,7 +94,3 @@ def generate_bqetl_docs(out_file):
     # append to bqetl docs page
     with open(out_file, "a") as out:
         out.write(output)
-
-
-if __name__ == "__main__":
-    generate_bqetl_docs()

bigquery_etl/docs/bqetl/templates/commands.md

@@ -1,63 +1,40 @@
+{% macro command_docs(cmd, n) -%}
+{{ "#" * n }} `{{ cmd.name }}`
+
+{{ cmd.description }}
+
+{% if cmd.commands | length > 0 -%}
+{% for command in cmd.commands -%}
+    {{ command_docs(command, n + 1) }}
+{% endfor -%}
+{% else -%}
+**Usage**
+
+```bash
+$ ./bqetl{{ cmd.path }} [OPTIONS]  
+{%- for arg in cmd.arguments -%}
+{{ "" }} [{{ arg.name }}]
+{%- endfor %}
+
+{% if cmd.options | length > 1 -%}
+Options:
+
+{% for option in cmd.options -%}
+--{{ option.name }}: {{ option.description }}
+{% endfor -%}
+{% endif -%}
+```
+
+{% if cmd.examples -%}
+**Examples**
+
+```bash
+{{ cmd.examples }}
+```
+{% endif -%}
+{% endif -%}
+{% endmacro -%}
+
 {% for command_group in command_groups -%}
-### `{{command_group.name}}`
-
-{{command_group.description}}
-
-{% for command in command_group.commands -%}
-#### `{{command.name}}`
-
-{{ command.description }}
-
-##### Usage
-
-```bash
-$ ./bqetl {{ command_group.name }} {{ command.name }} [OPTIONS]  
-{%- for arg in command.arguments -%}
-  {{ "" }} [{{ arg.name }}]
-{%- endfor %}
-
-{% if command.options | length > 1 -%}
-Options:
-
-{% for option in command.options -%}
-  --{{ option.name }}: {{ option.description }}
-{% endfor -%}
-{% endif %}
-```
-
-{% if command.examples -%}
-##### Examples
-
-```bash
-{{ command.examples }}
-```
-{% endif -%}
-{% endfor -%}
-
-{% if command_group.commands | length == 0 -%}
-#### Usage
-
-```bash
-$ ./bqetl {{ command_group.name }} [OPTIONS] 
-{%- for arg in command_group.arguments -%}
-  {{ "" }}[{{ arg.name }}]
-{%- endfor %}
-
-{% if command_group.options | length > 1 -%}
-Options:
-
-{% for option in command_group.options -%}
-  --{{ option.name }}: {{ option.description }}
-{% endfor -%}
-{% endif -%}
-```
-
-{% if command_group.examples -%}
-#### Examples
-
-```bash
-{{ command_group.examples }}
-```
-{% endif -%}
-{% endif -%}
+    {{ command_docs(command_group, 3) }}
 {% endfor -%}

docs/bqetl.md

@@ -4,7 +4,7 @@ The `bqetl` command-line tool aims to simplify working with the bigquery-etl rep
 
 ## Installation
 
-Please follow the [Quick Start](https://github.com/mozilla/bigquery-etl#quick-start) for setting up bigquery-etl and the bqetl CLI.
+Follow the [Quick Start](https://github.com/mozilla/bigquery-etl#quick-start) to set up bigquery-etl and the bqetl CLI.
 
 ## Commands
 

docs/mkdocs.yml

@@ -34,12 +34,5 @@ nav:
       - ... | mozdata/**.md
   - UDFs:
       - ... | mozfun/**.md
-    - index.md
-    - Cookbooks:
-        - Creating a derived dataset: cookbooks/creating_a_derived_dataset.md
-    - Datasets:
-        - ... | mozdata/**.md
-    - UDFs:
-        - ... | mozfun/**.md
-    - Reference:
-        - bqetl CLI: bqetl.md
+  - Reference:
+      - bqetl CLI: bqetl.md