diff --git a/taskcluster/taskgraph/transforms/job/mozharness_test.py b/taskcluster/taskgraph/transforms/job/mozharness_test.py index 5f546c440810..c0a5a8a42dd4 100644 --- a/taskcluster/taskgraph/transforms/job/mozharness_test.py +++ b/taskcluster/taskgraph/transforms/job/mozharness_test.py @@ -383,3 +383,86 @@ def mozharness_test_on_native_engine(config, job, taskdesc): download_symbols = mozharness['download-symbols'] download_symbols = {True: 'true', False: 'false'}.get(download_symbols, download_symbols) command.append('--download-symbols=' + download_symbols) + + +@run_job_using('script-engine-autophone', 'mozharness-test', schema=mozharness_test_run_schema) +def mozharness_test_on_script_engine_autophone(config, job, taskdesc): + test = taskdesc['run']['test'] + mozharness = test['mozharness'] + worker = taskdesc['worker'] + is_talos = test['suite'] == 'talos' + if worker['os'] != 'linux': + raise Exception('os: {} not supported on script-engine-autophone'.format(worker['os'])) + + installer_url = get_artifact_url('', mozharness['build-artifact-name']) + mozharness_url = get_artifact_url('', + 'public/build/mozharness.zip') + + artifacts = [ + # (artifact name prefix, in-image path) + ("public/test/", "/builds/worker/artifacts"), + ("public/logs/", "/builds/worker/workspace/build/upload/logs"), + ("public/test_info/", "/builds/worker/workspace/build/blobber_upload_dir"), + ] + + worker['artifacts'] = [{ + 'name': prefix, + 'path': path, + 'type': 'directory', + } for (prefix, path) in artifacts] + + if test['reboot']: + worker['reboot'] = test['reboot'] + + worker['env'] = env = { + 'GECKO_HEAD_REPOSITORY': config.params['head_repository'], + 'GECKO_HEAD_REV': config.params['head_rev'], + 'MOZHARNESS_CONFIG': ' '.join(mozharness['config']), + 'MOZHARNESS_SCRIPT': mozharness['script'], + 'MOZHARNESS_URL': {'task-reference': mozharness_url}, + 'MOZILLA_BUILD_URL': {'task-reference': installer_url}, + "MOZ_NO_REMOTE": '1', + "NO_EM_RESTART": '1', + "XPCOM_DEBUG_BREAK": 'warn', + "NO_FAIL_ON_TEST_ERRORS": '1', + "MOZ_HIDE_RESULTS_TABLE": '1', + "MOZ_NODE_PATH": "/usr/local/bin/node", + 'MOZ_AUTOMATION': '1', + 'WORKSPACE': '/builds/worker/workspace', + 'TASKCLUSTER_WORKER_TYPE': job['worker-type'], + + } + # talos tests don't need Xvfb + if is_talos: + env['NEED_XVFB'] = 'false' + + script = 'test-linux.sh' + worker['context'] = '{}/raw-file/{}/taskcluster/scripts/tester/{}'.format( + config.params['head_repository'], config.params['head_rev'], script + ) + + command = worker['command'] = ["./{}".format(script)] + command.extend([ + {"task-reference": "--installer-url=" + installer_url}, + {"task-reference": "--test-packages-url=" + test_packages_url(taskdesc)}, + ]) + if mozharness.get('include-blob-upload-branch'): + command.append('--blob-upload-branch=' + config.params['project']) + command.extend(mozharness.get('extra-options', [])) + + # TODO: remove the need for run['chunked'] + if mozharness.get('chunked') or test['chunks'] > 1: + # Implement mozharness['chunking-args'], modifying command in place + if mozharness['chunking-args'] == 'this-chunk': + command.append('--total-chunk={}'.format(test['chunks'])) + command.append('--this-chunk={}'.format(test['this-chunk'])) + elif mozharness['chunking-args'] == 'test-suite-suffix': + suffix = mozharness['chunk-suffix'].replace('', str(test['this-chunk'])) + for i, c in enumerate(command): + if isinstance(c, basestring) and c.startswith('--test-suite'): + command[i] += suffix + + if 'download-symbols' in mozharness: + download_symbols = mozharness['download-symbols'] + download_symbols = {True: 'true', False: 'false'}.get(download_symbols, download_symbols) + command.append('--download-symbols=' + download_symbols) diff --git a/taskcluster/taskgraph/transforms/task.py b/taskcluster/taskgraph/transforms/task.py index 5aa4a0a0421d..042259530b73 100644 --- a/taskcluster/taskgraph/transforms/task.py +++ b/taskcluster/taskgraph/transforms/task.py @@ -397,6 +397,36 @@ task_description_schema = Schema({ # environment variables Optional('env'): {basestring: taskref_or_string}, + # artifacts to extract from the task image after completion + Optional('artifacts'): [{ + # type of artifact -- simple file, or recursive directory + Required('type'): Any('file', 'directory'), + + # task image path from which to read artifact + Required('path'): basestring, + + # name of the produced artifact (root of the names for + # type=directory) + Required('name'): basestring, + }], + }, { + Required('implementation'): 'script-engine-autophone', + Required('os'): Any('macosx', 'linux'), + + # A link for an executable to download + Optional('context'): basestring, + + # Tells the worker whether machine should reboot + # after the task is finished. + Optional('reboot'): + Any(False, 'always', 'never', 'on-exception', 'on-failure'), + + # the command to run + Optional('command'): [taskref_or_string], + + # environment variables + Optional('env'): {basestring: taskref_or_string}, + # artifacts to extract from the task image after completion Optional('artifacts'): [{ # type of artifact -- simple file, or recursive directory @@ -1210,6 +1240,29 @@ def build_macosx_engine_payload(config, task, task_def): raise Exception('needs-sccache not supported in native-engine') +@payload_builder('script-engine-autophone') +def build_script_engine_autophone_payload(config, task, task_def): + worker = task['worker'] + artifacts = map(lambda artifact: { + 'name': artifact['name'], + 'path': artifact['path'], + 'type': artifact['type'], + 'expires': task_def['expires'], + }, worker.get('artifacts', [])) + + task_def['payload'] = { + 'context': worker['context'], + 'command': worker['command'], + 'env': worker['env'], + 'artifacts': artifacts, + } + if worker.get('reboot'): + task_def['payload'] = worker['reboot'] + + if task.get('needs-sccache'): + raise Exception('needs-sccache not supported in taskcluster-worker') + + transforms = TransformSequence() diff --git a/taskcluster/taskgraph/transforms/tests.py b/taskcluster/taskgraph/transforms/tests.py index 8d5ff9e24bce..ee23c70474d2 100644 --- a/taskcluster/taskgraph/transforms/tests.py +++ b/taskcluster/taskgraph/transforms/tests.py @@ -559,6 +559,8 @@ def set_treeherder_machine_platform(config, tests): # the table, workaround the issue for QR. if '-qr' in test['test-platform']: test['treeherder-machine-platform'] = test['test-platform'] + elif 'android-hw' in test['test-platform']: + test['treeherder-machine-platform'] = test['test-platform'] else: test['treeherder-machine-platform'] = translation.get( test['build-platform'], test['test-platform']) @@ -685,6 +687,8 @@ def handle_suite_category(config, tests): pass elif script == 'android_emulator_unittest.py': category_arg = '--test-suite' + elif script == 'android_hardware_unittest.py': + category_arg = '--test-suite' elif script == 'desktop_unittest.py': category_arg = '--{}-suite'.format(suite) @@ -928,6 +932,16 @@ def set_worker_type(config, tests): ] # now we have the right platform set the worker type accordingly test['worker-type'] = win_worker_type_platform[test['virtualization']] + elif test_platform.startswith('android-hw-g5'): + if test['suite'] == 'raptor': + test['worker-type'] = 'proj-autophone/gecko-t-ap-perf-g5' + else: + test['worker-type'] = 'proj-autophone/gecko-t-ap-unit-g5' + elif test_platform.startswith('android-hw-p2'): + if test['suite'] == 'raptor': + test['worker-type'] = 'proj-autophone/gecko-t-ap-perf-p2' + else: + test['worker-type'] = 'proj-autophone/gecko-t-ap-unit-p2' elif test_platform.startswith('linux') or test_platform.startswith('android'): if test.get('suite', '') == 'talos' and \ not test['build-platform'].startswith('linux64-ccov'): diff --git a/taskcluster/taskgraph/try_option_syntax.py b/taskcluster/taskgraph/try_option_syntax.py index 2c4b94f768df..c9beae1ec7bd 100644 --- a/taskcluster/taskgraph/try_option_syntax.py +++ b/taskcluster/taskgraph/try_option_syntax.py @@ -133,6 +133,9 @@ UNITTEST_PLATFORM_PRETTY_NAMES = { 'linux64-stylo-sequential' ], 'Android 4.3 Emulator': ['android-em-4.3-arm7-api-16'], + 'Android 7.0 Moto G5 32bit': ['android-hw-g5-7.0-arm7-api-16'], + 'Android 8.0 Google Pixel 2 32bit': ['android-hw-p2-8.0-arm7-api-16'], + 'Android 8.0 Google Pixel 2 64bit': ['android-hw-p2-8.0-android-aarch64'], '10.10': ['macosx64'], # other commonly-used substrings for platforms not yet supported with # in-tree taskgraphs: diff --git a/taskcluster/taskgraph/util/workertypes.py b/taskcluster/taskgraph/util/workertypes.py index 4d019f688d06..c56b7048c7bd 100644 --- a/taskcluster/taskgraph/util/workertypes.py +++ b/taskcluster/taskgraph/util/workertypes.py @@ -46,6 +46,10 @@ WORKER_TYPES = { "scriptworker-prov-v1/shipit-dev": ('shipit', None), "scriptworker-prov-v1/treescript-v1": ('treescript', None), 'releng-hardware/gecko-t-osx-1010': ('generic-worker', 'macosx'), + 'proj-autophone/gecko-t-ap-perf-g5': ('script-engine-autophone', 'linux'), + 'proj-autophone/gecko-t-ap-unit-g5': ('script-engine-autophone', 'linux'), + 'proj-autophone/gecko-t-ap-perf-p2': ('script-engine-autophone', 'linux'), + 'proj-autophone/gecko-t-ap-unit-p2': ('script-engine-autophone', 'linux'), }