Bug 1609987 - Allow checking additional paths in verify_docs. r=tomprince

There's no way for Thunderbird to define a new task kind at the moment because
the verify_docs function only checks one base path.

This adds a 'documentation_paths' registry that holds a list of paths to check
for documentation files. Calling documentation_paths() with a filename later
will read all the files it finds with that name under all of the base paths,
returning them in one chunk. This minimizes the necessary changes to verify_docs
itself.

Differential Revision: https://phabricator.services.mozilla.com/D60298

--HG--
extra : moz-landing-system : lando

taskcluster/taskgraph/util/verify.py

@@ -16,7 +16,7 @@ from .. import GECKO
 from .treeherder import join_symbol
 
 logger = logging.getLogger(__name__)
-base_path = os.path.join(GECKO, 'taskcluster', 'docs')
+doc_base_path = os.path.join(GECKO, 'taskcluster', 'docs')
 
 
 @attr.s(frozen=True)
@@ -47,35 +47,69 @@ class VerificationSequence(object):
 verifications = VerificationSequence()
 
 
-def verify_docs(filename, identifiers, appearing_as):
+@attr.s(frozen=True)
+class DocPaths(object):
+    _paths = attr.ib(factory=list)
 
+    def get_files(self, filename):
+        rv = []
+        for p in self._paths:
+            doc_path = os.path.join(p, filename)
+            if os.path.exists(doc_path):
+                rv.append(doc_path)
+        return rv
+
+    def add(self, path):
+        """
+        Projects that make use of Firefox's taskgraph can extend it with
+        their own task kinds by registering additional paths for documentation.
+        documentation_paths.add() needs to be called by the project's Taskgraph
+        registration function. See taskgraph.config.
+        """
+        self._paths.append(path)
+
+
+documentation_paths = DocPaths()
+documentation_paths.add(doc_base_path)
+
+
+def verify_docs(filename, identifiers, appearing_as):
+    """
+    Look for identifiers of the type appearing_as in the files
+    returned by documentation_paths.get_files(). Firefox will have
+    a single file in a list, but projects such as Thunderbird can have
+    documentation in another location and may return multiple files.
+    """
     # We ignore identifiers starting with '_' for the sake of tests.
     # Strings starting with "_" are ignored for doc verification
     # hence they can be used for faking test values
-    with open(os.path.join(base_path, filename)) as fileObject:
-        doctext = "".join(fileObject.readlines())
-        if appearing_as == "inline-literal":
-            expression_list = [
-                "``" + identifier + "``"
-                for identifier in identifiers
-                if not identifier.startswith("_")
-            ]
-        elif appearing_as == "heading":
-            expression_list = [
-                '\n' + identifier + "\n(?:(?:(?:-+\n)+)|(?:(?:.+\n)+))"
-                for identifier in identifiers
-                if not identifier.startswith("_")
-            ]
-        else:
-            raise Exception("appearing_as = `{}` not defined".format(appearing_as))
+    doc_files = documentation_paths.get_files(filename)
+    doctext = "".join(
+        [open(d).read() for d in doc_files]
+    )
 
-        for expression, identifier in zip(expression_list, identifiers):
-            match_group = re.search(expression, doctext)
-            if not match_group:
-                raise Exception(
-                    "{}: `{}` missing from doc file: `{}`"
-                    .format(appearing_as, identifier, filename)
-                )
+    if appearing_as == "inline-literal":
+        expression_list = [
+            "``" + identifier + "``"
+            for identifier in identifiers
+            if not identifier.startswith("_")
+        ]
+    elif appearing_as == "heading":
+        expression_list = [
+            '\n' + identifier + "\n(?:(?:(?:-+\n)+)|(?:(?:.+\n)+))"
+            for identifier in identifiers
+            if not identifier.startswith("_")
+        ]
+    else:
+        raise Exception("appearing_as = `{}` not defined".format(appearing_as))
+
+    for expression, identifier in zip(expression_list, identifiers):
+        match_group = re.search(expression, doctext)
+        if not match_group:
+            raise Exception(
+                "{}: `{}` missing from doc file: `{}`"
+                .format(appearing_as, identifier, filename)
+            )
 
 
 @verifications.add('full_task_graph')