From 28c3011f518cc20284325990f5d4f7cf2e59f646 Mon Sep 17 00:00:00 2001 From: Gregory Mierzwinski Date: Thu, 14 Jan 2021 13:54:29 +0000 Subject: [PATCH] Bug 1686118 - Always run downstream tasks on browsertime test retriggers. r=jmaher,bhearsum This patch makes it so that any retrigger actions being performed from Treeherder on browsertime tasks (through add-new-jobs, retrigger, and retrigger-multiple) will always retrigger the vismet (visual-metrics) tasks correctly. Furthermore, when a vismet task is retriggered, we'll retrigger the browsertime test task. Differential Revision: https://phabricator.services.mozilla.com/D101404 --- taskcluster/taskgraph/actions/add_new_jobs.py | 19 ++++++++- taskcluster/taskgraph/actions/retrigger.py | 41 +++++++++++++++---- taskcluster/taskgraph/actions/util.py | 13 ++++++ 3 files changed, 65 insertions(+), 8 deletions(-) diff --git a/taskcluster/taskgraph/actions/add_new_jobs.py b/taskcluster/taskgraph/actions/add_new_jobs.py index 4a15821e648d..76d13b22c5b0 100644 --- a/taskcluster/taskgraph/actions/add_new_jobs.py +++ b/taskcluster/taskgraph/actions/add_new_jobs.py @@ -12,6 +12,8 @@ from .util import ( combine_task_graph_files, create_tasks, fetch_graph_and_labels, + get_tasks_with_downstream, + rename_browsertime_vismet_task, ) @@ -47,11 +49,26 @@ def add_new_jobs_action(parameters, graph_config, input, task_group_id, task_id) ) to_run = [] + browsertime_tasks = [] for elem in input["tasks"]: if elem in full_task_graph.tasks: - to_run.append(elem) + if "browsertime" in elem: + label = elem + if "vismet" in label: + label = rename_browsertime_vismet_task(label) + browsertime_tasks.append(label) + else: + to_run.append(elem) else: raise Exception("{} was not found in the task-graph".format(elem)) + if len(browsertime_tasks) > 0: + to_run.extend( + list( + get_tasks_with_downstream( + browsertime_tasks, full_task_graph, label_to_taskid + ) + ) + ) times = input.get("times", 1) for i in range(times): diff --git a/taskcluster/taskgraph/actions/retrigger.py b/taskcluster/taskgraph/actions/retrigger.py index 79905931f92e..5aa53135b25b 100644 --- a/taskcluster/taskgraph/actions/retrigger.py +++ b/taskcluster/taskgraph/actions/retrigger.py @@ -17,6 +17,8 @@ from .util import ( fetch_graph_and_labels, relativize_datestamps, create_task_from_def, + get_tasks_with_downstream, + rename_browsertime_vismet_task, ) from .registry import register_callback_action from taskgraph.util import taskcluster @@ -150,6 +152,10 @@ def retrigger_action(parameters, graph_config, input, task_group_id, task_id): task = taskcluster.get_task_definition(task_id) label = task["metadata"]["name"] + force_downstream = "browsertime" in label + if "vismet" in label: + label = rename_browsertime_vismet_task(label) + with_downstream = " " to_run = [label] @@ -160,11 +166,8 @@ def retrigger_action(parameters, graph_config, input, task_group_id, task_id): ) sys.exit(1) - if input.get("downstream"): - to_run = full_task_graph.graph.transitive_closure( - set(to_run), reverse=True - ).nodes - to_run = to_run & set(label_to_taskid.keys()) + if input.get("downstream") or force_downstream: + to_run = get_tasks_with_downstream(to_run, full_task_graph, label_to_taskid) with_downstream = " (with downstream) " times = input.get("times", 1) @@ -266,18 +269,41 @@ def retrigger_multiple(parameters, graph_config, input, task_group_id, task_id): suffixes = [] for i, request in enumerate(input.get("requests", [])): + + def _is_browsertime(label): + return "browsertime" in label + times = request.get("times", 1) rerun_tasks = [ label for label in request.get("tasks") if not _should_retrigger(full_task_graph, label) + and not _is_browsertime(label) ] retrigger_tasks = [ label for label in request.get("tasks") - if _should_retrigger(full_task_graph, label) + if _should_retrigger(full_task_graph, label) and not _is_browsertime(label) ] + browsertime_tasks = [] + for label in request.get("tasks"): + if not _is_browsertime(label): + continue + if "vismet" in label: + label = rename_browsertime_vismet_task(label) + browsertime_tasks.append(label) + + # Browsertime tasks need to have their downstream tasks scheduled as well + if len(browsertime_tasks) > 0: + retrigger_tasks.extend( + list( + get_tasks_with_downstream( + browsertime_tasks, full_task_graph, label_to_taskid + ) + ) + ) + for label in rerun_tasks: # XXX we should not re-run tasks pulled in from other pushes # In practice, this shouldn't matter, as only completed tasks @@ -298,4 +324,5 @@ def retrigger_multiple(parameters, graph_config, input, task_group_id, task_id): suffix, ) - combine_task_graph_files(suffixes) + if suffixes: + combine_task_graph_files(suffixes) diff --git a/taskcluster/taskgraph/actions/util.py b/taskcluster/taskgraph/actions/util.py index d97a27f1f2a2..8b0e9845a0ed 100644 --- a/taskcluster/taskgraph/actions/util.py +++ b/taskcluster/taskgraph/actions/util.py @@ -151,6 +151,19 @@ def get_parameters(decision_task_id): return get_artifact(decision_task_id, "public/parameters.yml") +def get_tasks_with_downstream(labels, full_task_graph, label_to_taskid): + # Used to gather tasks when downstream tasks need to run as well + return full_task_graph.graph.transitive_closure( + set(labels), reverse=True + ).nodes & set(label_to_taskid.keys()) + + +def rename_browsertime_vismet_task(label): + # Vismet tasks have labels which are modified from + # the task label which created the data so we can undo it here + return label.replace("-vismet", "") + "-e10s" + + def fetch_graph_and_labels(parameters, graph_config): decision_task_id = find_decision_task(parameters, graph_config)