Bug 1652086: Make `kind_dependencies_tasks` a dictionary based on the label; r=Callek

I was looking at py-spy for generating the taskgraph, and found that a bunch
of time was spent in taskgraph.transforms.job.use_fetches[1]. Use a dictionary
there instead saves about 20-30s on my machine.

[1] https://searchfox.org/mozilla-central/rev/622dbd3409610ad3f71b56c9a6a92da905dab0aa/taskcluster/taskgraph/transforms/job/__init__.py#243-247

Differential Revision: https://phabricator.services.mozilla.com/D83166

taskcluster/taskgraph/generator.py

@@ -54,8 +54,8 @@ class Kind(object):
         config = copy.deepcopy(self.config)
 
         kind_dependencies = config.get('kind-dependencies', [])
-        kind_dependencies_tasks = [task for task in loaded_tasks
-                                   if task.kind in kind_dependencies]
+        kind_dependencies_tasks = {task.label: task for task in loaded_tasks
+                                   if task.kind in kind_dependencies}
 
         inputs = loader(self.name, self.path, config, parameters, loaded_tasks)
 

taskcluster/taskgraph/transforms/cached_tasks.py

@@ -54,7 +54,7 @@ def cache_task(config, tasks):
         return
 
     digests = {}
-    for task in config.kind_dependencies_tasks:
+    for task in config.kind_dependencies_tasks.values():
         if 'cached_task' in task.attributes:
             digests[task.label] = format_task_digest(task.attributes['cached_task'])
 

taskcluster/taskgraph/transforms/code_review.py

@@ -18,7 +18,7 @@ def add_dependencies(config, jobs):
         job.setdefault('soft-dependencies', [])
         job['soft-dependencies'] += [
             dep_task.label
-            for dep_task in config.kind_dependencies_tasks
+            for dep_task in config.kind_dependencies_tasks.values()
             if dep_task.attributes.get('code-review') is True
         ]
         yield job

taskcluster/taskgraph/transforms/docker_image.py

@@ -82,13 +82,6 @@ transforms.add_validate(docker_image_schema)
 
 @transforms.add
 def fill_template(config, tasks):
-    available_packages = set()
-    for task in config.kind_dependencies_tasks:
-        if task.kind != 'packages':
-            continue
-        name = task.label.replace('packages-', '')
-        available_packages.add(name)
-
     if not taskgraph.fast and config.write_artifacts:
         if not os.path.isdir(CONTEXTS_DIR):
             os.makedirs(CONTEXTS_DIR)
@@ -102,7 +95,7 @@ def fill_template(config, tasks):
         parent = task.pop('parent', None)
 
         for p in packages:
-            if p not in available_packages:
+            if "packages-{}".format(p) not in config.kind_dependencies_tasks:
                 raise Exception('Missing package job for {}-{}: {}'.format(
                     config.kind, image_name, p))
 

taskcluster/taskgraph/transforms/job/__init__.py

@@ -187,7 +187,7 @@ def use_fetches(config, jobs):
             if value:
                 aliases['{}-{}'.format(config.kind, value)] = label
 
-    for task in config.kind_dependencies_tasks:
+    for task in config.kind_dependencies_tasks.values():
         if task.kind in ('fetch', 'toolchain'):
             get_attribute(
                 artifact_names, task.label, task.attributes,
@@ -240,25 +240,17 @@ def use_fetches(config, jobs):
                 if dep_label in artifact_prefixes:
                     prefix = artifact_prefixes[dep_label]
                 else:
-                    dep_tasks = [
-                        task
-                        for task in config.kind_dependencies_tasks
-                        if task.label == dep_label
-                    ]
-                    if len(dep_tasks) != 1:
+                    if dep_label not in config.kind_dependencies_tasks:
                         raise Exception(
                             "{name} can't fetch {kind} artifacts because "
-                            "there are {tasks} with label {label} in kind dependencies!".format(
+                            "there are no tasks with label {label} in kind dependencies!".format(
                                 name=name,
                                 kind=kind,
                                 label=dependencies[kind],
-                                tasks="no tasks"
-                                if len(dep_tasks) == 0
-                                else "multiple tasks",
                             )
                         )
 
-                    prefix = get_artifact_prefix(dep_tasks[0])
+                    prefix = get_artifact_prefix(config.kind_dependencies_tasks[dep_label])
 
                 for artifact in artifacts:
                     if isinstance(artifact, text_type):

taskcluster/taskgraph/transforms/release_deps.py

@@ -25,7 +25,7 @@ def add_dependencies(config, jobs):
             continue
 
         required_signoffs = set(job.setdefault('attributes', {}).get('required_signoffs', []))
-        for dep_task in config.kind_dependencies_tasks:
+        for dep_task in config.kind_dependencies_tasks.values():
             # Weed out unwanted tasks.
             # XXX we have run-on-projects which specifies the on-push behavior;
             # we need another attribute that specifies release promotion,

taskcluster/taskgraph/transforms/update_verify.py

@@ -18,7 +18,7 @@ transforms = TransformSequence()
 @transforms.add
 def add_command(config, tasks):
     config_tasks = {}
-    for dep in config.kind_dependencies_tasks:
+    for dep in config.kind_dependencies_tasks.values():
         if 'update-verify-config' in dep.kind or 'update-verify-next-config' in dep.kind:
             config_tasks[dep.name] = dep