Strip temporary dir name from filepath for docs generation (#1819)

* Remove "tmp/generated-sql" from dir name

Since docs are generated from 'tmp/generated-sql' dir,
we need to remove 'tmp/generated-sql' from file names
so they can be checked against the SKIP list

* Defide more specific destination path for mozfun

* Add pages sorting to mozfun

bigquery_etl/docs/generate_docs.py

@@ -129,7 +129,9 @@ def main():
                         if UDF_FILE in files or PROCEDURE_FILE in files:
                             # UDF-level doc; append to dataset doc
                             dataset_name = os.path.basename(path)
-                            dataset_doc = out_dir / path.parent / f"{dataset_name}.md"
+                            dataset_doc = (
+                                Path(out_dir) / "mozfun" / f"{dataset_name}.md"
+                            )
                             docfile_content = load_with_examples(src)
                             with open(dataset_doc, "a") as dataset_doc_file:
                                 dataset_doc_file.write("\n\n")
@@ -150,7 +152,7 @@ def main():
                                 dataset_doc_file.write(f"{sourced}\n\n")
                         else:
                             # dataset-level doc; create a new doc file
-                            dest = out_dir / path / f"{name}.md"
+                            dest = Path(out_dir) / "mozfun" / f"{name}.md"
                             dest.write_text(load_with_examples(src))
                 else:
                     generate_derived_dataset_docs.generate_derived_dataset_docs(

bigquery_etl/dryrun.py

@@ -240,10 +240,14 @@ class DryRun:
 
     def get_referenced_tables(self):
         """Return referenced tables by dry running the SQL file."""
-        if self.sqlfile not in SKIP and not self.is_valid():
+        # strip file path to 'sql/project/dataset/table/*.sql' format
+        # to be checked against the SKIP list
+        filepath = self.sqlfile[self.sqlfile.find("/sql/") + 1 :]
+
+        if filepath not in SKIP and not self.is_valid():
             raise Exception(f"Error when dry running SQL file {self.sqlfile}")
 
-        if self.sqlfile in SKIP:
+        if filepath in SKIP:
             print(f"\t...Ignoring dryrun results for {self.sqlfile}")
 
         if (

docs/mozfun/.pages

@@ -0,0 +1,3 @@
+nav:
+- mozfun.md
+- ...