From ff5e584d9d8c7a0fea78a6a1f3ed42cfdd9e6c68 Mon Sep 17 00:00:00 2001 From: Armen Zambrano Gasparnian Date: Thu, 28 Jul 2016 13:20:44 -0400 Subject: [PATCH] Bug 1279676 - Support --rebuild try flag to schedule tests N times. r=dustin MozReview-Commit-ID: Lrxi8t53nwy If a developer adds '--rebuild N' to their try syntax they will get test jobs scheduled N times. This is useful to determine intermittency rate. This fixes a regression due to the recent refactoring on how we schedule tasks. --HG-- extra : rebase_source : 355ca631353015bf63461c194168d753efd6958e --- taskcluster/docs/attributes.rst | 13 +++++++++++++ taskcluster/taskgraph/create.py | 7 +++++++ taskcluster/taskgraph/target_tasks.py | 13 +++++++++++-- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/taskcluster/docs/attributes.rst b/taskcluster/docs/attributes.rst index b34e5126a9d1..52f4ef59824a 100644 --- a/taskcluster/docs/attributes.rst +++ b/taskcluster/docs/attributes.rst @@ -19,6 +19,19 @@ kind A task's ``kind`` attribute gives the name of the kind that generated it, e.g., ``build`` or ``legacy``. +task_duplicates +=============== + +This is used to indicate that we want multiple copies of the task created. +This feature is used to track down intermittent job failures. + +If this value is set to N, the task-creation machinery will create a total of N +copies of the task. Only the first copy will be included in the taskgraph +output artifacts, although all tasks will be contained in the same taskGroup. + +While most attributes are considered read-only, target task methods may alter +this attribute of tasks they include in the target set. + build_platform ============== diff --git a/taskcluster/taskgraph/create.py b/taskcluster/taskgraph/create.py index 770b8669a9e6..a1968f740e4f 100644 --- a/taskcluster/taskgraph/create.py +++ b/taskcluster/taskgraph/create.py @@ -54,6 +54,7 @@ def create_tasks(taskgraph, label_to_taskid): # that. for task_id in taskgraph.graph.visit_postorder(): task_def = taskgraph.tasks[task_id].task + attributes = taskgraph.tasks[task_id].attributes # if this task has no dependencies, make it depend on this decision # task so that it does not start immediately; and so that if this loop # fails halfway through, none of the already-created tasks run. @@ -72,6 +73,12 @@ def create_tasks(taskgraph, label_to_taskid): fs[task_id] = e.submit(_create_task, session, task_id, taskid_to_label[task_id], task_def) + # Schedule tasks as many times as task_duplicates indicates + for i in range(1, attributes.get('task_duplicates', 1)): + # We use slugid() since we want a distinct task id + fs[task_id] = e.submit(_create_task, session, slugid(), + taskid_to_label[task_id], task_def) + # Wait for all futures to complete. for f in futures.as_completed(fs.values()): f.result() diff --git a/taskcluster/taskgraph/target_tasks.py b/taskcluster/taskgraph/target_tasks.py index b6445afad953..b7f4294fa378 100644 --- a/taskcluster/taskgraph/target_tasks.py +++ b/taskcluster/taskgraph/target_tasks.py @@ -42,8 +42,17 @@ def target_tasks_try_option_syntax(full_task_graph, parameters): """Generate a list of target tasks based on try syntax in parameters['message'] and, for context, the full task graph.""" options = try_option_syntax.TryOptionSyntax(parameters['message'], full_task_graph) - return [t.label for t in full_task_graph.tasks.itervalues() - if options.task_matches(t.attributes)] + target_tasks_labels = [t.label for t in full_task_graph.tasks.itervalues() + if options.task_matches(t.attributes)] + + # If the developer wants test jobs to be rebuilt N times we add that value here + if int(options.trigger_tests) > 1: + for l in target_tasks_labels: + task = full_task_graph[l] + if 'unittest_suite' in task.attributes: + task.attributes['task_duplicates'] = options.trigger_tests + + return target_tasks_labels @_target_task('all_builds_and_tests')