Bug 1280570 - Retrigger tc tasks when mozharness returns TBPL_RETRY; r=dustin

taskcluster/taskgraph/transforms/job/mozharness.py

@@ -140,6 +140,9 @@ def mozharness_on_docker_worker_setup(config, job, taskdesc):
         env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool'
         env['TOOLTOOL_REV'] = 'master'
 
+    # Retry if mozharness returns TBPL_RETRY
+    worker['retry-exit-status'] = 4
+
     docker_worker_setup_secrets(config, job, taskdesc)
 
     command = [

taskcluster/taskgraph/transforms/task.py

@@ -197,6 +197,10 @@ task_description_schema = Schema({
 
         # the maximum time to run, in seconds
         'max-run-time': int,
+
+        # the exit status code that indicates the task should be retried
+        Optional('retry-exit-status'): int,
+
     }, {
         Required('implementation'): 'generic-worker',
 
@@ -348,6 +352,9 @@ def build_docker_worker_payload(config, task, task_def):
     if 'max-run-time' in worker:
         payload['maxRunTime'] = worker['max-run-time']
 
+    if 'retry-exit-status' in worker:
+        payload['onExitStatus'] = {'retry': [worker['retry-exit-status']]}
+
     if 'artifacts' in worker:
         artifacts = {}
         for artifact in worker['artifacts']:
@@ -398,6 +405,9 @@ def build_generic_worker_payload(config, task, task_def):
         'maxRunTime': worker['max-run-time'],
     }
 
+    if 'retry-exit-status' in worker:
+        raise Exception("retry-exit-status not supported in generic-worker")
+
 
 transforms = TransformSequence()
 

taskcluster/taskgraph/transforms/tests/all_kinds.py

@@ -122,3 +122,12 @@ def split_chunks(config, tests):
             chunked['treeherder-symbol'] = join_symbol(group, symbol)
 
             yield chunked
+
+
+@transforms.add
+def set_retry_exit_status(config, tests):
+    """Set the retry exit status to TBPL_RETRY, the value returned by mozharness
+       scripts to indicate a transient failure that should be retried."""
+    for test in tests:
+        test['retry-exit-status'] = 4
+        yield test

taskcluster/taskgraph/transforms/tests/make_task_description.py

@@ -146,6 +146,7 @@ def docker_worker_setup(config, test, taskdesc):
     worker['loopback-video'] = test['loopback-video']
     worker['loopback-audio'] = test['loopback-audio']
     worker['max-run-time'] = test['max-run-time']
+    worker['retry-exit-status'] = test['retry-exit-status']
 
     worker['artifacts'] = [{
         'name': prefix,

taskcluster/taskgraph/transforms/tests/test_description.py

@@ -130,6 +130,9 @@ test_description_schema = Schema({
         {'by-test-platform': {basestring: int}},
     ),
 
+    # the exit status code that indicates the task should be retried
+    Optional('retry-exit-status'): int,
+
     # Whether to perform a gecko checkout.
     Required('checkout', default=False): bool,
 

testing/mozharness/mozharness/mozilla/testing/errors.py

@@ -98,7 +98,7 @@ TinderBoxPrintRe = {
     "harness_error": {
         'full_regex': re.compile(r"(?:TEST-UNEXPECTED-FAIL|PROCESS-CRASH) \| .* \| (application crashed|missing output line for total leaks!|negative leaks caught!|\d+ bytes leaked)"),
         'minimum_regex': re.compile(r'''(TEST-UNEXPECTED|PROCESS-CRASH)'''),
-        'retry_regex': re.compile(r'''FAIL-SHOULD-RETRY''')
+        'retry_regex': re.compile(r'''(FAIL-SHOULD-RETRY|No space left on device|DMError|Connection to the other side was lost in a non-clean fashion|program finished with exit code 80|INFRA-ERROR|twisted.spread.pb.PBConnectionLost)''')
     },
 }