diff --git a/taskcluster/scripts/run-task b/taskcluster/scripts/run-task index ddb090f9d8b7..8f4ea64797af 100755 --- a/taskcluster/scripts/run-task +++ b/taskcluster/scripts/run-task @@ -448,13 +448,14 @@ def vcs_checkout(source_repo, dest, store_path, if sparse_profile: args.extend(['--sparseprofile', sparse_profile]) + dest = os.path.abspath(dest) args.extend([ revision_flag, revision_value, source_repo, dest, ]) res = run_and_prefix_output(b'vcs', args, - extra_env={b'PYTHONUNBUFFERED': b'1'}) + extra_env={'PYTHONUNBUFFERED': '1'}) if res: sys.exit(res) @@ -741,6 +742,9 @@ def main(args): return 1 try: + if 'GECKO_PATH' in os.environ: + os.environ['GECKO_PATH'] = os.path.abspath(os.environ['GECKO_PATH']) + if 'MOZ_FETCHES' in os.environ: fetch_artifacts() diff --git a/taskcluster/taskgraph/transforms/job/common.py b/taskcluster/taskgraph/transforms/job/common.py index 6f3e599028ac..c01f4cc7bb7a 100644 --- a/taskcluster/taskgraph/transforms/job/common.py +++ b/taskcluster/taskgraph/transforms/job/common.py @@ -64,10 +64,20 @@ def support_vcs_checkout(config, job, taskdesc, sparse=False): This can only be used with ``run-task`` tasks, as the cache name is reserved for ``run-task`` tasks. """ - level = config.params['level'] + is_win = job['worker']['os'] == 'windows' - # native-engine does not support caches (yet), so we just do a full clone - # every time :( + if is_win: + checkoutdir = './build' + geckodir = '{}/src'.format(checkoutdir) + hgstore = 'y:/hg-shared' + else: + checkoutdir = '{workdir}/checkouts'.format(**job['run']) + geckodir = '{}/gecko'.format(checkoutdir) + hgstore = '{}/hg-store'.format(checkoutdir) + + level = config.params['level'] + # native-engine and generic-worker do not support caches (yet), so we just + # do a full clone every time :( if job['worker']['implementation'] in ('docker-worker', 'docker-engine'): name = 'level-%s-checkouts' % level @@ -84,15 +94,15 @@ def support_vcs_checkout(config, job, taskdesc, sparse=False): taskdesc['worker'].setdefault('caches', []).append({ 'type': 'persistent', 'name': name, - 'mount-point': '{workdir}/checkouts'.format(**job['run']), + 'mount-point': checkoutdir, }) taskdesc['worker'].setdefault('env', {}).update({ 'GECKO_BASE_REPOSITORY': config.params['base_repository'], 'GECKO_HEAD_REPOSITORY': config.params['head_repository'], 'GECKO_HEAD_REV': config.params['head_rev'], - 'GECKO_PATH': '{workdir}/checkouts/gecko'.format(**job['run']), - 'HG_STORE_PATH': '{workdir}/checkouts/hg-store'.format(**job['run']), + 'GECKO_PATH': geckodir, + 'HG_STORE_PATH': hgstore, }) if 'comm_base_repository' in config.params: diff --git a/taskcluster/taskgraph/transforms/job/mach.py b/taskcluster/taskgraph/transforms/job/mach.py index a216d263a141..6b643588b5a7 100644 --- a/taskcluster/taskgraph/transforms/job/mach.py +++ b/taskcluster/taskgraph/transforms/job/mach.py @@ -30,14 +30,19 @@ mach_schema = Schema({ }) -@run_job_using("docker-worker", "mach", schema=mach_schema, defaults={'comm-checkout': False}) -@run_job_using("native-engine", "mach", schema=mach_schema, defaults={'comm-checkout': False}) -@run_job_using("generic-worker", "mach", schema=mach_schema, defaults={'comm-checkout': False}) -def docker_worker_mach(config, job, taskdesc): +defaults = { + 'comm-checkout': False, +} + + +@run_job_using("docker-worker", "mach", schema=mach_schema, defaults=defaults) +@run_job_using("native-engine", "mach", schema=mach_schema, defaults=defaults) +@run_job_using("generic-worker", "mach", schema=mach_schema, defaults=defaults) +def configure_mach(config, job, taskdesc): run = job['run'] # defer to the run_task implementation - run['command'] = 'cd {workdir}/checkouts/gecko && ./mach {mach}'.format(**run) + run['command'] = 'cd $GECKO_PATH && ./mach {mach}'.format(**run) run['using'] = 'run-task' del run['mach'] configure_taskdesc_for_run(config, job, taskdesc, job['worker']['implementation']) diff --git a/taskcluster/taskgraph/transforms/job/python_test.py b/taskcluster/taskgraph/transforms/job/python_test.py index eaac16532300..10dafb225ba4 100644 --- a/taskcluster/taskgraph/transforms/job/python_test.py +++ b/taskcluster/taskgraph/transforms/job/python_test.py @@ -25,12 +25,15 @@ python_test_schema = Schema({ }) -@run_job_using( - 'docker-worker', - 'python-test', - schema=python_test_schema, - defaults={'python-version': 2, 'subsuite': 'default'}) -def docker_worker_python_test(config, job, taskdesc): +defaults = { + 'python-version': 2, + 'subsuite': 'default', +} + + +@run_job_using('docker-worker', 'python-test', schema=python_test_schema, defaults=defaults) +@run_job_using('generic-worker', 'python-test', schema=python_test_schema, defaults=defaults) +def configure_python_test(config, job, taskdesc): run = job['run'] # defer to the mach implementation diff --git a/taskcluster/taskgraph/transforms/job/run_task.py b/taskcluster/taskgraph/transforms/job/run_task.py index be6b1b0c67a3..800355414fba 100644 --- a/taskcluster/taskgraph/transforms/job/run_task.py +++ b/taskcluster/taskgraph/transforms/job/run_task.py @@ -41,12 +41,12 @@ run_task_schema = Schema({ }) -def common_setup(config, job, taskdesc, command, checkoutdir): +def common_setup(config, job, taskdesc, command, geckodir): run = job['run'] if run['checkout']: support_vcs_checkout(config, job, taskdesc, sparse=bool(run['sparse-profile'])) - command.append('--vcs-checkout={}/gecko'.format(checkoutdir)) + command.append('--vcs-checkout={}'.format(geckodir)) if run['sparse-profile']: command.append('--sparse-profile=build/sparse-profiles/%s' % @@ -73,7 +73,8 @@ def docker_worker_run_task(config, job, taskdesc): run = job['run'] worker = taskdesc['worker'] = job['worker'] command = ['/builds/worker/bin/run-task'] - common_setup(config, job, taskdesc, command, checkoutdir='{workdir}/checkouts'.format(**run)) + common_setup(config, job, taskdesc, command, + geckodir='{workdir}/checkouts/gecko'.format(**run)) if run.get('cache-dotcache'): worker['caches'].append({ @@ -100,7 +101,8 @@ def native_engine_run_task(config, job, taskdesc): run = job['run'] worker = taskdesc['worker'] = job['worker'] command = ['./run-task'] - common_setup(config, job, taskdesc, command, checkoutdir='{workdir}/checkouts'.format(**run)) + common_setup(config, job, taskdesc, command, + geckodir='{workdir}/checkouts/gecko'.format(**run)) worker['context'] = run_task_url(config) @@ -119,8 +121,16 @@ def native_engine_run_task(config, job, taskdesc): def generic_worker_run_task(config, job, taskdesc): run = job['run'] worker = taskdesc['worker'] = job['worker'] - command = ['./run-task'] - common_setup(config, job, taskdesc, command, checkoutdir='{workdir}/checkouts'.format(**run)) + is_win = worker['os'] == 'windows' + + if is_win: + command = ['C:/mozilla-build/python3/python3.exe', 'run-task'] + geckodir = './build/src' + else: + command = ['./run-task'] + geckodir = '{workdir}/checkouts/gecko'.format(**run) + + common_setup(config, job, taskdesc, command, geckodir=geckodir) worker.setdefault('mounts', []) if run.get('cache-dotcache'): @@ -137,7 +147,17 @@ def generic_worker_run_task(config, job, taskdesc): run_command = run['command'] if isinstance(run_command, basestring): + if is_win: + run_command = '"{}"'.format(run_command) run_command = ['bash', '-cx', run_command] + command.append('--') command.extend(run_command) - worker['command'] = [['chmod', '+x', 'run-task'], command] + + if is_win: + worker['command'] = [' '.join(command)] + else: + worker['command'] = [ + ['chmod', '+x', 'run-task'], + command, + ] diff --git a/taskcluster/taskgraph/transforms/task.py b/taskcluster/taskgraph/transforms/task.py index 5246218aff44..31353e96bf49 100644 --- a/taskcluster/taskgraph/transforms/task.py +++ b/taskcluster/taskgraph/transforms/task.py @@ -1880,8 +1880,8 @@ def check_caches_are_volumes(task): return raise Exception('task %s (image %s) has caches that are not declared as ' - 'Docker volumes: %s' - 'Have you added them as VOLUMEs in the Dockerfile?' + 'Docker volumes: %s ' + '(have you added them as VOLUMEs in the Dockerfile?)' % (task['label'], task['worker']['docker-image'], ', '.join(sorted(missing))))