Bug 1748929 - Move TaskGraphGenerator.verify_* to a verifications in util/verify.py, r=releng-reviewers,aki

Standalone taskgraph doesn't have a 'verify_parameters' function as we can't
assume that parameters are documented in every consumer repo. By moving these
to proper verifications in util/verify.py, each consumer repo in standalone may
optional implement something for them or not (as they see fit).

Differential Revision: https://phabricator.services.mozilla.com/D142024

taskcluster/gecko_taskgraph/generator.py

@@ -19,10 +19,7 @@ from .morph import morph
 from .optimize import optimize_task_graph
 from .transforms.base import TransformSequence, TransformConfig
 from .util.python_path import find_object
-from .util.verify import (
-    verify_docs,
-    verifications,
-)
+from .util.verify import verifications
 from .config import load_graph_config
 
 logger = logging.getLogger(__name__)
@@ -252,6 +249,9 @@ class TaskGraphGenerator:
                     continue
 
     def _run(self):
+        # Initial verifications that don't depend on any generation state.
+        verifications("initial")
+
         logger.info("Loading graph configuration.")
         graph_config = load_graph_config(self.root_dir)
 
@@ -263,7 +263,6 @@ class TaskGraphGenerator:
             parameters = self._parameters(graph_config)
         else:
             parameters = self._parameters
-        self.verify_parameters(parameters)
 
         logger.info("Using {}".format(parameters))
         logger.debug("Dumping parameters:\n{}".format(repr(parameters)))
@@ -274,7 +273,7 @@ class TaskGraphGenerator:
             filters.insert(0, "target_tasks_method")
         filters = [filter_tasks.filter_task_functions[f] for f in filters]
 
-        yield ("parameters", parameters)
+        yield self.verify("parameters", parameters)
 
         logger.info("Loading kinds")
         # put the kinds into a graph and sort topologically so that kinds are loaded
@@ -290,7 +289,7 @@ class TaskGraphGenerator:
             kind.name: kind
             for kind in self._load_kinds(graph_config, parameters.get("target-kind"))
         }
-        self.verify_kinds(kinds)
+        verifications("kinds", kinds)
 
         edges = set()
         for kind in kinds.values():
@@ -321,11 +320,7 @@ class TaskGraphGenerator:
                 all_tasks[task.label] = task
             logger.info(f"Generated {len(new_tasks)} tasks for kind {kind_name}")
         full_task_set = TaskGraph(all_tasks, Graph(set(all_tasks), set()))
-        self.verify_attributes(all_tasks)
-        self.verify_run_using()
-        yield self.verify_graph(
-            "full_task_set", full_task_set, graph_config, parameters
-        )
+        yield self.verify("full_task_set", full_task_set, graph_config, parameters)
 
         logger.info("Generating full task graph")
         edges = set()
@@ -338,9 +333,7 @@ class TaskGraphGenerator:
             "Full task graph contains %d tasks and %d dependencies"
             % (len(full_task_set.graph.nodes), len(edges))
         )
-        yield self.verify_graph(
-            "full_task_graph", full_task_graph, graph_config, parameters
-        )
+        yield self.verify("full_task_graph", full_task_graph, graph_config, parameters)
 
         logger.info("Generating target task set")
         target_task_set = TaskGraph(
@@ -357,9 +350,7 @@ class TaskGraphGenerator:
                 % (fltr.__name__, old_len - len(target_tasks), len(target_tasks))
             )
 
-        yield self.verify_graph(
-            "target_task_set", target_task_set, graph_config, parameters
-        )
+        yield self.verify("target_task_set", target_task_set, graph_config, parameters)
 
         logger.info("Generating target task graph")
         # include all docker-image build tasks here, in case they are needed for a graph morph
@@ -386,7 +377,7 @@ class TaskGraphGenerator:
         target_task_graph = TaskGraph(
             {l: all_tasks[l] for l in target_graph.nodes}, target_graph
         )
-        yield self.verify_graph(
+        yield self.verify(
             "target_task_graph", target_task_graph, graph_config, parameters
         )
 
@@ -413,7 +404,7 @@ class TaskGraphGenerator:
             strategy_override=strategies,
         )
 
-        yield self.verify_graph(
+        yield self.verify(
             "optimized_task_graph", optimized_task_graph, graph_config, parameters
         )
 
@@ -426,7 +417,7 @@ class TaskGraphGenerator:
         )
 
         yield "label_to_taskid", label_to_taskid
-        yield self.verify_graph(
+        yield self.verify(
             "morphed_task_graph", morphed_task_graph, graph_config, parameters
         )
 
@@ -439,44 +430,9 @@ class TaskGraphGenerator:
             self._run_results[k] = v
         return self._run_results[name]
 
-    def verify_graph(self, name, graph, graph_config, parameters):
-        verifications(name, graph, graph_config, parameters)
-        return name, graph
-
-    def verify_parameters(self, parameters):
-        if not parameters.strict:
-            return
-
-        parameters_dict = dict(**parameters)
-        verify_docs(
-            filename="parameters.rst",
-            identifiers=list(parameters_dict),
-            appearing_as="inline-literal",
-        )
-
-    def verify_kinds(self, kinds):
-        verify_docs(
-            filename="kinds.rst", identifiers=kinds.keys(), appearing_as="heading"
-        )
-
-    def verify_attributes(self, all_tasks):
-        attribute_set = set()
-        for label, task in all_tasks.items():
-            attribute_set.update(task.attributes.keys())
-        verify_docs(
-            filename="attributes.rst",
-            identifiers=list(attribute_set),
-            appearing_as="heading",
-        )
-
-    def verify_run_using(self):
-        from .transforms.job import registry
-
-        verify_docs(
-            filename="transforms.rst",
-            identifiers=registry.keys(),
-            appearing_as="inline-literal",
-        )
+    def verify(self, name, obj, *args, **kwargs):
+        verifications(name, obj, *args, **kwargs)
+        return name, obj
 
 
 def load_tasks_for_kind(parameters, kind, root_dir=None):

taskcluster/gecko_taskgraph/util/verify.py

@@ -37,8 +37,18 @@ class Verification(ABC):
         pass
 
 
+@attr.s(frozen=True)
+class InitialVerification(Verification):
+    """Verification that doesn't depend on any generation state."""
+
+    def verify(self):
+        self.func()
+
+
 @attr.s(frozen=True)
 class GraphVerification(Verification):
+    """Verification for a TaskGraph object."""
+
     run_on_projects = attr.ib(default=None)
 
     def verify(
@@ -65,6 +75,22 @@ class GraphVerification(Verification):
         )
 
 
+@attr.s(frozen=True)
+class ParametersVerification(Verification):
+    """Verification for a set of parameters."""
+
+    def verify(self, parameters: Parameters):
+        self.func(parameters)
+
+
+@attr.s(frozen=True)
+class KindsVerification(Verification):
+    """Verification for kinds."""
+
+    def verify(self, kinds: dict):
+        self.func(kinds)
+
+
 @attr.s(frozen=True)
 class VerificationSequence:
     """
@@ -78,6 +104,9 @@ class VerificationSequence:
     _verifications = attr.ib(factory=dict)
     _verification_types = {
         "graph": GraphVerification,
+        "initial": InitialVerification,
+        "kinds": KindsVerification,
+        "parameters": ParametersVerification,
     }
 
     def __call__(self, name, *args, **kwargs):
@@ -161,6 +190,47 @@ def verify_docs(filename, identifiers, appearing_as):
             )
 
 
+@verifications.add("initial")
+def verify_run_using():
+    from gecko_taskgraph.transforms.job import registry
+
+    verify_docs(
+        filename="transforms.rst",
+        identifiers=registry.keys(),
+        appearing_as="inline-literal",
+    )
+
+
+@verifications.add("parameters")
+def verify_parameters_docs(parameters):
+    if not parameters.strict:
+        return
+
+    parameters_dict = dict(**parameters)
+    verify_docs(
+        filename="parameters.rst",
+        identifiers=list(parameters_dict),
+        appearing_as="inline-literal",
+    )
+
+
+@verifications.add("kinds")
+def verify_kinds_docs(kinds):
+    verify_docs(filename="kinds.rst", identifiers=kinds.keys(), appearing_as="heading")
+
+
+@verifications.add("full_task_set")
+def verify_attributes(task, taskgraph, scratch_pad, graph_config, parameters):
+    if task is None:
+        verify_docs(
+            filename="attributes.rst",
+            identifiers=list(scratch_pad["attribute_set"]),
+            appearing_as="heading",
+        )
+        return
+    scratch_pad.setdefault("attribute_set", set()).update(task.attributes.keys())
+
+
 @verifications.add("full_task_graph")
 def verify_task_graph_symbol(task, taskgraph, scratch_pad, graph_config, parameters):
     """