diff --git a/taskcluster/ci/build/macosx.yml b/taskcluster/ci/build/macosx.yml index a45a0b8ab858..5f0c4b8f906f 100644 --- a/taskcluster/ci/build/macosx.yml +++ b/taskcluster/ci/build/macosx.yml @@ -23,7 +23,7 @@ macosx64/debug: tooltool-downloads: internal macosx64/opt: - description: "MacOS X x64 Cross-compile" + description: "MacOS X x64" index: product: firefox job-name: macosx64-opt @@ -31,15 +31,14 @@ macosx64/opt: platform: osx-10-7/opt symbol: tc(B) tier: 2 - worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64 + worker-type: buildbot-bridge/buildbot-bridge worker: - implementation: docker-worker - max-run-time: 36000 + implementation: buildbot-bridge run: using: mozharness actions: [get-secrets build generate-build-stats update] config: - - builds/releng_base_mac_64_cross_builds.py + - builds/releng_base_mac_64_builds.py - balrog/production.py script: "mozharness/scripts/fx_desktop_build.py" secrets: true diff --git a/taskcluster/taskgraph/morph.py b/taskcluster/taskgraph/morph.py index e9ad8b34de1f..55918f424cc0 100644 --- a/taskcluster/taskgraph/morph.py +++ b/taskcluster/taskgraph/morph.py @@ -157,7 +157,92 @@ def add_index_tasks(taskgraph, label_to_taskid): return taskgraph, label_to_taskid +def make_s3_uploader_task(parent_task): + if parent_task.task['payload']['sourcestamp']['branch'] == 'try': + worker_type = 'buildbot-try' + else: + worker_type = 'buildbot' + + task_def = { + # The null-provisioner and buildbot worker type don't actually exist. + # So this task doesn't actually run - we just need to create the task so + # we have something to attach artifacts to. + "provisionerId": "null-provisioner", + "workerType": worker_type, + "created": {'relative-datestamp': '0 seconds'}, + "deadline": parent_task.task['deadline'], + "routes": [], + "payload": {}, + "metadata": { + "name": "Buildbot/mozharness S3 uploader", + "description": "Upload outputs of buildbot/mozharness builds to S3", + "owner": "mshal@mozilla.com", + "source": "http://hg.mozilla.org/build/mozharness/", + } + } + label = 's3-uploader-{}'.format(parent_task.label) + dependencies = {} + task = Task(kind='misc', label=label, attributes={}, task=task_def, + dependencies=dependencies) + task.task_id = parent_task.task['payload']['properties']['upload_to_task_id'] + return task + + +def update_test_tasks(taskid, build_taskid, taskgraph): + """Tests task must download artifacts from uploader task. + + Notice they don't need to depend on uploader task because it finishes + before the build task. + """ + # Notice we handle buildbot-bridge, native, and generic-worker payloads + # We can do better here in terms of graph searching + # We could do post order search and stop as soon as we + # reach the build task. Not worring about it because this is + # (supposed to be) a temporary solution. + for task in taskgraph.tasks.itervalues(): + if build_taskid in task.task.get('dependencies', []): + payload = task.task['payload'] + if 'command' in payload: + try: + payload['command'] = [ + cmd.replace(build_taskid, taskid) for cmd in payload['command'] + ] + except AttributeError: + # generic-worker command attribute is an list of lists + payload['command'] = [ + [cmd.replace(build_taskid, taskid) for cmd in x] + for x in payload['command'] + ] + if 'mounts' in payload: + for mount in payload['mounts']: + if mount.get('content', {}).get('taskId', '') == build_taskid: + mount['content']['taskId'] = taskid + if 'env' in payload: + payload['env'] = { + k: v.replace(build_taskid, taskid) for k, v in payload['env'].iteritems() + } + if 'properties' in payload: + payload['properties']['parent_task_id'] = taskid + + +def add_s3_uploader_task(taskgraph, label_to_taskid): + """The S3 uploader task is used by mozharness to upload buildbot artifacts.""" + for task in taskgraph.tasks.itervalues(): + if 'upload_to_task_id' in task.task.get('payload', {}).get('properties', {}): + added = make_s3_uploader_task(task) + taskgraph, label_to_taskid = amend_taskgraph( + taskgraph, label_to_taskid, [added]) + update_test_tasks(added.task_id, task.task_id, taskgraph) + logger.info('Added s3-uploader task') + return taskgraph, label_to_taskid + + def morph(taskgraph, label_to_taskid): """Apply all morphs""" - taskgraph, label_to_taskid = add_index_tasks(taskgraph, label_to_taskid) + morphs = [ + add_index_tasks, + add_s3_uploader_task, + ] + for m in morphs: + taskgraph, label_to_taskid = m(taskgraph, label_to_taskid) return taskgraph, label_to_taskid diff --git a/taskcluster/taskgraph/transforms/job/mozharness.py b/taskcluster/taskgraph/transforms/job/mozharness.py index 0894faca6174..3a80b6cbdabb 100644 --- a/taskcluster/taskgraph/transforms/job/mozharness.py +++ b/taskcluster/taskgraph/transforms/job/mozharness.py @@ -9,6 +9,7 @@ way, and certainly anything using mozharness should use this approach. """ from __future__ import absolute_import, print_function, unicode_literals +import slugid from textwrap import dedent @@ -244,3 +245,27 @@ def mozharness_on_generic_worker(config, job, taskdesc): ' '.join(hg_command), ' '.join(mh_command) ]) + + +@run_job_using('buildbot-bridge', 'mozharness', schema=mozharness_run_schema) +def mozharness_on_buildbot_bridge(config, job, taskdesc): + run = job['run'] + worker = taskdesc['worker'] + branch = config.params['project'] + product = run.get('index', {}).get('product', 'firefox') + + worker.pop('env', None) + + worker.update({ + 'buildername': 'OS X 10.7 {} build'.format(branch), + 'sourcestamp': { + 'branch': branch, + 'repository': config.params['head_repository'], + 'revision': config.params['head_rev'], + }, + 'properties': { + 'product': product, + 'who': config.params['owner'], + 'upload_to_task_id': slugid.nice(), + } + }) diff --git a/taskcluster/taskgraph/transforms/job/mozharness_test.py b/taskcluster/taskgraph/transforms/job/mozharness_test.py index ad12dfac2ad0..59a0ee32af6b 100644 --- a/taskcluster/taskgraph/transforms/job/mozharness_test.py +++ b/taskcluster/taskgraph/transforms/job/mozharness_test.py @@ -210,17 +210,25 @@ def mozharness_test_on_generic_worker(config, job, taskdesc): 'path': 'logs/log_warning.log', 'type': 'file' }, - { + ] + + # jittest doesn't have blob_upload_dir + if test['test-name'] != 'jittest': + artifacts.append({ 'name': 'public/test_info', 'path': 'build/blobber_upload_dir', 'type': 'directory' - } - ] + }) build_platform = taskdesc['attributes']['build_platform'] + build_type = taskdesc['attributes']['build_type'] - target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), build_platform) \ - if build_platform.startswith('win') else 'target' + if build_platform.startswith('win'): + target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), build_platform) + elif build_platform == 'macosx64' and build_type == 'opt': + target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), 'mac') + else: + target = 'target' installer_url = get_artifact_url('', mozharness['build-artifact-name']) @@ -330,9 +338,14 @@ def mozharness_test_on_native_engine(config, job, taskdesc): mozharness = test['mozharness'] worker = taskdesc['worker'] + build_platform = taskdesc['attributes']['build_platform'] + build_type = taskdesc['attributes']['build_type'] + target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), 'mac') \ + if build_platform == 'macosx64' and build_type == 'opt' else 'target' + installer_url = get_artifact_url('', mozharness['build-artifact-name']) test_packages_url = get_artifact_url('', - 'public/build/target.test_packages.json') + 'public/build/{}.test_packages.json'.format(target)) mozharness_url = get_artifact_url('', 'public/build/mozharness.zip') diff --git a/taskcluster/taskgraph/transforms/tests.py b/taskcluster/taskgraph/transforms/tests.py index f5efac7e1477..0347bf78b3b5 100644 --- a/taskcluster/taskgraph/transforms/tests.py +++ b/taskcluster/taskgraph/transforms/tests.py @@ -346,7 +346,13 @@ def set_target(config, tests): for test in tests: build_platform = test['build-platform'] if build_platform.startswith('macosx'): - target = 'target.dmg' + if build_platform.split('/')[1] == 'opt': + target = 'firefox-{}.en-US.{}.dmg'.format( + get_firefox_version(), + 'mac', + ) + else: + target = 'target.dmg' elif build_platform.startswith('android'): if 'geckoview' in test['test-name']: target = 'geckoview_example.apk' diff --git a/testing/mozharness/mozharness/mozilla/building/buildbase.py b/testing/mozharness/mozharness/mozilla/building/buildbase.py index fb87667c9352..379c56b4d582 100755 --- a/testing/mozharness/mozharness/mozilla/building/buildbase.py +++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py @@ -1419,6 +1419,7 @@ or run without that action (ie: --no-{action})" routes.append(template.format(**fmt)) self.info("Using routes: %s" % routes) + taskid = self.buildbot_config['properties'].get('upload_to_task_id') tc = Taskcluster( branch=self.branch, rank=pushinfo.pushdate, # Use pushdate as the rank @@ -1427,10 +1428,13 @@ or run without that action (ie: --no-{action})" log_obj=self.log_obj, # `upload_to_task_id` is used by mozci to have access to where the artifacts # will be uploaded - task_id=self.buildbot_config['properties'].get('upload_to_task_id'), + task_id=taskid, ) - task = tc.create_task(routes) + if taskid: + task = tc.get_task(taskid) + else: + task = tc.create_task(routes) tc.claim_task(task) # Only those files uploaded with valid extensions are processed.