Bug 1426586 - Add job transforms for android hardware tests, r=jonasfj

* * *
Bug 1426586 - Add worker type proj-autophone/gecko-t-ap-unit-{g5,p2} for android-hw-common-tests, r=gbrown,jmaher

taskcluster/taskgraph/transforms/job/mozharness_test.py

@@ -383,3 +383,86 @@ def mozharness_test_on_native_engine(config, job, taskdesc):
         download_symbols = mozharness['download-symbols']
         download_symbols = {True: 'true', False: 'false'}.get(download_symbols, download_symbols)
         command.append('--download-symbols=' + download_symbols)
+
+
+@run_job_using('script-engine-autophone', 'mozharness-test', schema=mozharness_test_run_schema)
+def mozharness_test_on_script_engine_autophone(config, job, taskdesc):
+    test = taskdesc['run']['test']
+    mozharness = test['mozharness']
+    worker = taskdesc['worker']
+    is_talos = test['suite'] == 'talos'
+    if worker['os'] != 'linux':
+        raise Exception('os: {} not supported on script-engine-autophone'.format(worker['os']))
+
+    installer_url = get_artifact_url('<build>', mozharness['build-artifact-name'])
+    mozharness_url = get_artifact_url('<build>',
+                                      'public/build/mozharness.zip')
+
+    artifacts = [
+        # (artifact name prefix, in-image path)
+        ("public/test/", "/builds/worker/artifacts"),
+        ("public/logs/", "/builds/worker/workspace/build/upload/logs"),
+        ("public/test_info/", "/builds/worker/workspace/build/blobber_upload_dir"),
+    ]
+
+    worker['artifacts'] = [{
+        'name': prefix,
+        'path': path,
+        'type': 'directory',
+    } for (prefix, path) in artifacts]
+
+    if test['reboot']:
+        worker['reboot'] = test['reboot']
+
+    worker['env'] = env = {
+        'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
+        'GECKO_HEAD_REV': config.params['head_rev'],
+        'MOZHARNESS_CONFIG': ' '.join(mozharness['config']),
+        'MOZHARNESS_SCRIPT': mozharness['script'],
+        'MOZHARNESS_URL': {'task-reference': mozharness_url},
+        'MOZILLA_BUILD_URL': {'task-reference': installer_url},
+        "MOZ_NO_REMOTE": '1',
+        "NO_EM_RESTART": '1',
+        "XPCOM_DEBUG_BREAK": 'warn',
+        "NO_FAIL_ON_TEST_ERRORS": '1',
+        "MOZ_HIDE_RESULTS_TABLE": '1',
+        "MOZ_NODE_PATH": "/usr/local/bin/node",
+        'MOZ_AUTOMATION': '1',
+        'WORKSPACE': '/builds/worker/workspace',
+        'TASKCLUSTER_WORKER_TYPE': job['worker-type'],
+
+    }
+    # talos tests don't need Xvfb
+    if is_talos:
+        env['NEED_XVFB'] = 'false'
+
+    script = 'test-linux.sh'
+    worker['context'] = '{}/raw-file/{}/taskcluster/scripts/tester/{}'.format(
+        config.params['head_repository'], config.params['head_rev'], script
+    )
+
+    command = worker['command'] = ["./{}".format(script)]
+    command.extend([
+        {"task-reference": "--installer-url=" + installer_url},
+        {"task-reference": "--test-packages-url=" + test_packages_url(taskdesc)},
+    ])
+    if mozharness.get('include-blob-upload-branch'):
+        command.append('--blob-upload-branch=' + config.params['project'])
+    command.extend(mozharness.get('extra-options', []))
+
+    # TODO: remove the need for run['chunked']
+    if mozharness.get('chunked') or test['chunks'] > 1:
+        # Implement mozharness['chunking-args'], modifying command in place
+        if mozharness['chunking-args'] == 'this-chunk':
+            command.append('--total-chunk={}'.format(test['chunks']))
+            command.append('--this-chunk={}'.format(test['this-chunk']))
+        elif mozharness['chunking-args'] == 'test-suite-suffix':
+            suffix = mozharness['chunk-suffix'].replace('<CHUNK>', str(test['this-chunk']))
+            for i, c in enumerate(command):
+                if isinstance(c, basestring) and c.startswith('--test-suite'):
+                    command[i] += suffix
+
+    if 'download-symbols' in mozharness:
+        download_symbols = mozharness['download-symbols']
+        download_symbols = {True: 'true', False: 'false'}.get(download_symbols, download_symbols)
+        command.append('--download-symbols=' + download_symbols)

taskcluster/taskgraph/transforms/task.py

@@ -397,6 +397,36 @@ task_description_schema = Schema({
         # environment variables
         Optional('env'): {basestring: taskref_or_string},
 
+        # artifacts to extract from the task image after completion
+        Optional('artifacts'): [{
+            # type of artifact -- simple file, or recursive directory
+            Required('type'): Any('file', 'directory'),
+
+            # task image path from which to read artifact
+            Required('path'): basestring,
+
+            # name of the produced artifact (root of the names for
+            # type=directory)
+            Required('name'): basestring,
+        }],
+    }, {
+        Required('implementation'): 'script-engine-autophone',
+        Required('os'): Any('macosx', 'linux'),
+
+        # A link for an executable to download
+        Optional('context'): basestring,
+
+        # Tells the worker whether machine should reboot
+        # after the task is finished.
+        Optional('reboot'):
+            Any(False, 'always', 'never', 'on-exception', 'on-failure'),
+
+        # the command to run
+        Optional('command'): [taskref_or_string],
+
+        # environment variables
+        Optional('env'): {basestring: taskref_or_string},
+
         # artifacts to extract from the task image after completion
         Optional('artifacts'): [{
             # type of artifact -- simple file, or recursive directory
@@ -1210,6 +1240,29 @@ def build_macosx_engine_payload(config, task, task_def):
         raise Exception('needs-sccache not supported in native-engine')
 
 
+@payload_builder('script-engine-autophone')
+def build_script_engine_autophone_payload(config, task, task_def):
+    worker = task['worker']
+    artifacts = map(lambda artifact: {
+        'name': artifact['name'],
+        'path': artifact['path'],
+        'type': artifact['type'],
+        'expires': task_def['expires'],
+    }, worker.get('artifacts', []))
+
+    task_def['payload'] = {
+        'context': worker['context'],
+        'command': worker['command'],
+        'env': worker['env'],
+        'artifacts': artifacts,
+    }
+    if worker.get('reboot'):
+        task_def['payload'] = worker['reboot']
+
+    if task.get('needs-sccache'):
+        raise Exception('needs-sccache not supported in taskcluster-worker')
+
+
 transforms = TransformSequence()
 
 

taskcluster/taskgraph/transforms/tests.py

@@ -559,6 +559,8 @@ def set_treeherder_machine_platform(config, tests):
         # the table, workaround the issue for QR.
         if '-qr' in test['test-platform']:
             test['treeherder-machine-platform'] = test['test-platform']
+        elif 'android-hw' in test['test-platform']:
+            test['treeherder-machine-platform'] = test['test-platform']
         else:
             test['treeherder-machine-platform'] = translation.get(
                 test['build-platform'], test['test-platform'])
@@ -685,6 +687,8 @@ def handle_suite_category(config, tests):
             pass
         elif script == 'android_emulator_unittest.py':
             category_arg = '--test-suite'
+        elif script == 'android_hardware_unittest.py':
+            category_arg = '--test-suite'
         elif script == 'desktop_unittest.py':
             category_arg = '--{}-suite'.format(suite)
 
@@ -928,6 +932,16 @@ def set_worker_type(config, tests):
                 ]
             # now we have the right platform set the worker type accordingly
             test['worker-type'] = win_worker_type_platform[test['virtualization']]
+        elif test_platform.startswith('android-hw-g5'):
+            if test['suite'] == 'raptor':
+                test['worker-type'] = 'proj-autophone/gecko-t-ap-perf-g5'
+            else:
+                test['worker-type'] = 'proj-autophone/gecko-t-ap-unit-g5'
+        elif test_platform.startswith('android-hw-p2'):
+            if test['suite'] == 'raptor':
+                test['worker-type'] = 'proj-autophone/gecko-t-ap-perf-p2'
+            else:
+                test['worker-type'] = 'proj-autophone/gecko-t-ap-unit-p2'
         elif test_platform.startswith('linux') or test_platform.startswith('android'):
             if test.get('suite', '') == 'talos' and \
                  not test['build-platform'].startswith('linux64-ccov'):

taskcluster/taskgraph/try_option_syntax.py

@@ -133,6 +133,9 @@ UNITTEST_PLATFORM_PRETTY_NAMES = {
         'linux64-stylo-sequential'
     ],
     'Android 4.3 Emulator': ['android-em-4.3-arm7-api-16'],
+    'Android 7.0 Moto G5 32bit': ['android-hw-g5-7.0-arm7-api-16'],
+    'Android 8.0 Google Pixel 2 32bit': ['android-hw-p2-8.0-arm7-api-16'],
+    'Android 8.0 Google Pixel 2 64bit': ['android-hw-p2-8.0-android-aarch64'],
     '10.10': ['macosx64'],
     # other commonly-used substrings for platforms not yet supported with
     # in-tree taskgraphs:

taskcluster/taskgraph/util/workertypes.py

@@ -46,6 +46,10 @@ WORKER_TYPES = {
     "scriptworker-prov-v1/shipit-dev": ('shipit', None),
     "scriptworker-prov-v1/treescript-v1": ('treescript', None),
     'releng-hardware/gecko-t-osx-1010': ('generic-worker', 'macosx'),
+    'proj-autophone/gecko-t-ap-perf-g5': ('script-engine-autophone', 'linux'),
+    'proj-autophone/gecko-t-ap-unit-g5': ('script-engine-autophone', 'linux'),
+    'proj-autophone/gecko-t-ap-perf-p2': ('script-engine-autophone', 'linux'),
+    'proj-autophone/gecko-t-ap-unit-p2': ('script-engine-autophone', 'linux'),
 }