Limit the dependency count for the nightly alert task (#5043)

taskcluster/app_services_taskgraph/transforms/deps_complete.py

@@ -1,4 +1,9 @@
+from copy import deepcopy
+import itertools
+
+from taskgraph import MAX_DEPENDENCIES
 from taskgraph.transforms.base import TransformSequence
+
 transforms = TransformSequence()
 
 @transforms.add
@@ -60,3 +65,41 @@ def add_alert_routes(config, tasks):
             task['routes'].append("notify.{}.{}.on-failed".format(name, value))
         yield task
 
+# Transform that adjusts the dependencies to not exceed MAX_DEPENDENCIES
+#
+# This transform checks if the dependency count exceeds MAX_DEPENDENCIES.  If
+# so, it creates a child jobs with exactly MAX_DEPENDENCIES that the main task
+# can then depend on.
+#
+# This is separated out from the main transform since it depends on
+# taskgraph.transforms.job:transforms running first.
+#
+# This code is based off the reverse_chunk_deps transform from Gecko
+reverse_chunk = TransformSequence()
+
+def adjust_dependencies_child_job(orig_job, deps, count):
+    job = deepcopy(orig_job)
+    job["soft-dependencies"] = deps
+    job["label"] = "{} - {}".format(orig_job["label"], count)
+    del job["routes"] # don't send alerts for child jobs
+    return job
+
+@reverse_chunk.add
+def adjust_dependencies(config, jobs):
+    for job in jobs:
+        counter = itertools.count(1)
+        # sort for deterministic chunking
+        deps = sorted(job["soft-dependencies"])
+
+        while len(deps) > MAX_DEPENDENCIES:
+            # split off the first N deps
+            chunk, deps = deps[:MAX_DEPENDENCIES], deps[MAX_DEPENDENCIES:]
+            # create a separate job to handle them
+            child_job = adjust_dependencies_child_job(job, chunk, next(counter))
+            # Yield the child job and add a dependency to it
+            yield child_job
+            deps.append(child_job["label"])
+
+        # Yield the parent job with the rest of the deps
+        job["soft-dependencies"] = deps
+        yield job

taskcluster/ci/nightly-complete/kind.yml

@@ -12,6 +12,7 @@ kind-dependencies:
 transforms:
   - app_services_taskgraph.transforms.deps_complete:transforms
   - taskgraph.transforms.job:transforms
+  - app_services_taskgraph.transforms.deps_complete:reverse_chunk
   - taskgraph.transforms.task:transforms
 
 jobs: