Bug 1686118 - Always run downstream tasks on browsertime test retriggers. r=jmaher,bhearsum

This patch makes it so that any retrigger actions being performed from Treeherder on browsertime tasks (through add-new-jobs, retrigger, and retrigger-multiple) will always retrigger the vismet (visual-metrics) tasks correctly. Furthermore, when a vismet task is retriggered, we'll retrigger the browsertime test task.

Differential Revision: https://phabricator.services.mozilla.com/D101404

taskcluster/taskgraph/actions/add_new_jobs.py

@@ -12,6 +12,8 @@ from .util import (
     combine_task_graph_files,
     create_tasks,
     fetch_graph_and_labels,
+    get_tasks_with_downstream,
+    rename_browsertime_vismet_task,
 )
 
 
@@ -47,11 +49,26 @@ def add_new_jobs_action(parameters, graph_config, input, task_group_id, task_id)
     )
 
     to_run = []
+    browsertime_tasks = []
     for elem in input["tasks"]:
         if elem in full_task_graph.tasks:
-            to_run.append(elem)
+            if "browsertime" in elem:
+                label = elem
+                if "vismet" in label:
+                    label = rename_browsertime_vismet_task(label)
+                browsertime_tasks.append(label)
+            else:
+                to_run.append(elem)
         else:
             raise Exception("{} was not found in the task-graph".format(elem))
+    if len(browsertime_tasks) > 0:
+        to_run.extend(
+            list(
+                get_tasks_with_downstream(
+                    browsertime_tasks, full_task_graph, label_to_taskid
+                )
+            )
+        )
 
     times = input.get("times", 1)
     for i in range(times):

taskcluster/taskgraph/actions/retrigger.py

@@ -17,6 +17,8 @@ from .util import (
     fetch_graph_and_labels,
     relativize_datestamps,
     create_task_from_def,
+    get_tasks_with_downstream,
+    rename_browsertime_vismet_task,
 )
 from .registry import register_callback_action
 from taskgraph.util import taskcluster
@@ -150,6 +152,10 @@ def retrigger_action(parameters, graph_config, input, task_group_id, task_id):
     task = taskcluster.get_task_definition(task_id)
     label = task["metadata"]["name"]
 
+    force_downstream = "browsertime" in label
+    if "vismet" in label:
+        label = rename_browsertime_vismet_task(label)
+
     with_downstream = " "
     to_run = [label]
 
@@ -160,11 +166,8 @@ def retrigger_action(parameters, graph_config, input, task_group_id, task_id):
         )
         sys.exit(1)
 
-    if input.get("downstream"):
-        to_run = full_task_graph.graph.transitive_closure(
-            set(to_run), reverse=True
-        ).nodes
-        to_run = to_run & set(label_to_taskid.keys())
+    if input.get("downstream") or force_downstream:
+        to_run = get_tasks_with_downstream(to_run, full_task_graph, label_to_taskid)
         with_downstream = " (with downstream) "
 
     times = input.get("times", 1)
@@ -266,18 +269,41 @@ def retrigger_multiple(parameters, graph_config, input, task_group_id, task_id):
 
     suffixes = []
     for i, request in enumerate(input.get("requests", [])):
+
+        def _is_browsertime(label):
+            return "browsertime" in label
+
         times = request.get("times", 1)
         rerun_tasks = [
             label
             for label in request.get("tasks")
             if not _should_retrigger(full_task_graph, label)
+            and not _is_browsertime(label)
         ]
         retrigger_tasks = [
             label
             for label in request.get("tasks")
-            if _should_retrigger(full_task_graph, label)
+            if _should_retrigger(full_task_graph, label) and not _is_browsertime(label)
         ]
 
+        browsertime_tasks = []
+        for label in request.get("tasks"):
+            if not _is_browsertime(label):
+                continue
+            if "vismet" in label:
+                label = rename_browsertime_vismet_task(label)
+            browsertime_tasks.append(label)
+
+        # Browsertime tasks need to have their downstream tasks scheduled as well
+        if len(browsertime_tasks) > 0:
+            retrigger_tasks.extend(
+                list(
+                    get_tasks_with_downstream(
+                        browsertime_tasks, full_task_graph, label_to_taskid
+                    )
+                )
+            )
+
         for label in rerun_tasks:
             # XXX we should not re-run tasks pulled in from other pushes
             # In practice, this shouldn't matter, as only completed tasks
@@ -298,4 +324,5 @@ def retrigger_multiple(parameters, graph_config, input, task_group_id, task_id):
                 suffix,
             )
 
-    combine_task_graph_files(suffixes)
+    if suffixes:
+        combine_task_graph_files(suffixes)

taskcluster/taskgraph/actions/util.py

@@ -151,6 +151,19 @@ def get_parameters(decision_task_id):
     return get_artifact(decision_task_id, "public/parameters.yml")
 
 
+def get_tasks_with_downstream(labels, full_task_graph, label_to_taskid):
+    # Used to gather tasks when downstream tasks need to run as well
+    return full_task_graph.graph.transitive_closure(
+        set(labels), reverse=True
+    ).nodes & set(label_to_taskid.keys())
+
+
+def rename_browsertime_vismet_task(label):
+    # Vismet tasks have labels which are modified from
+    # the task label which created the data so we can undo it here
+    return label.replace("-vismet", "") + "-e10s"
+
+
 def fetch_graph_and_labels(parameters, graph_config):
     decision_task_id = find_decision_task(parameters, graph_config)