Bug 1350413 part 1: move macosx64/opt build to buildbot-bridge. r=dustin

Because of bug 1338651, we need to stick to BBB macosx64 builds for now.

MozReview-Commit-ID: AwQc5r6ikUN

--HG--
extra : rebase_source : d5ee1605bf5ec70223e7676bd2d2fd760841d59a

taskcluster/ci/build/macosx.yml

@@ -23,7 +23,7 @@ macosx64/debug:
         tooltool-downloads: internal
 
 macosx64/opt:
-    description: "MacOS X x64 Cross-compile"
+    description: "MacOS X x64"
     index:
         product: firefox
         job-name: macosx64-opt
@@ -31,15 +31,14 @@ macosx64/opt:
         platform: osx-10-7/opt
         symbol: tc(B)
         tier: 2
-    worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64
+    worker-type: buildbot-bridge/buildbot-bridge
     worker:
-        implementation: docker-worker
-        max-run-time: 36000
+        implementation: buildbot-bridge
     run:
         using: mozharness
         actions: [get-secrets build generate-build-stats update]
         config:
-            - builds/releng_base_mac_64_cross_builds.py
+            - builds/releng_base_mac_64_builds.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true

taskcluster/taskgraph/morph.py

@@ -157,7 +157,92 @@ def add_index_tasks(taskgraph, label_to_taskid):
     return taskgraph, label_to_taskid
 
 
+def make_s3_uploader_task(parent_task):
+    if parent_task.task['payload']['sourcestamp']['branch'] == 'try':
+        worker_type = 'buildbot-try'
+    else:
+        worker_type = 'buildbot'
+
+    task_def = {
+        # The null-provisioner and buildbot worker type don't actually exist.
+        # So this task doesn't actually run - we just need to create the task so
+        # we have something to attach artifacts to.
+        "provisionerId": "null-provisioner",
+        "workerType": worker_type,
+        "created": {'relative-datestamp': '0 seconds'},
+        "deadline": parent_task.task['deadline'],
+        "routes": [],
+        "payload": {},
+        "metadata": {
+            "name": "Buildbot/mozharness S3 uploader",
+            "description": "Upload outputs of buildbot/mozharness builds to S3",
+            "owner": "mshal@mozilla.com",
+            "source": "http://hg.mozilla.org/build/mozharness/",
+        }
+    }
+    label = 's3-uploader-{}'.format(parent_task.label)
+    dependencies = {}
+    task = Task(kind='misc', label=label, attributes={}, task=task_def,
+                dependencies=dependencies)
+    task.task_id = parent_task.task['payload']['properties']['upload_to_task_id']
+    return task
+
+
+def update_test_tasks(taskid, build_taskid, taskgraph):
+    """Tests task must download artifacts from uploader task.
+
+    Notice they don't need to depend on uploader task because it finishes
+    before the build task.
+    """
+    # Notice we handle buildbot-bridge, native, and generic-worker payloads
+    # We can do better here in terms of graph searching
+    # We could do post order search and stop as soon as we
+    # reach the build task. Not worring about it because this is
+    # (supposed to be) a temporary solution.
+    for task in taskgraph.tasks.itervalues():
+        if build_taskid in task.task.get('dependencies', []):
+            payload = task.task['payload']
+            if 'command' in payload:
+                try:
+                    payload['command'] = [
+                        cmd.replace(build_taskid, taskid) for cmd in payload['command']
+                    ]
+                except AttributeError:
+                    # generic-worker command attribute is an list of lists
+                    payload['command'] = [
+                        [cmd.replace(build_taskid, taskid) for cmd in x]
+                        for x in payload['command']
+                    ]
+            if 'mounts' in payload:
+                for mount in payload['mounts']:
+                    if mount.get('content', {}).get('taskId', '') == build_taskid:
+                        mount['content']['taskId'] = taskid
+            if 'env' in payload:
+                payload['env'] = {
+                    k: v.replace(build_taskid, taskid) for k, v in payload['env'].iteritems()
+                }
+            if 'properties' in payload:
+                payload['properties']['parent_task_id'] = taskid
+
+
+def add_s3_uploader_task(taskgraph, label_to_taskid):
+    """The S3 uploader task is used by mozharness to upload buildbot artifacts."""
+    for task in taskgraph.tasks.itervalues():
+        if 'upload_to_task_id' in task.task.get('payload', {}).get('properties', {}):
+            added = make_s3_uploader_task(task)
+            taskgraph, label_to_taskid = amend_taskgraph(
+                taskgraph, label_to_taskid, [added])
+            update_test_tasks(added.task_id, task.task_id, taskgraph)
+            logger.info('Added s3-uploader task')
+    return taskgraph, label_to_taskid
+
+
 def morph(taskgraph, label_to_taskid):
     """Apply all morphs"""
-    taskgraph, label_to_taskid = add_index_tasks(taskgraph, label_to_taskid)
+    morphs = [
+        add_index_tasks,
+        add_s3_uploader_task,
+    ]
+    for m in morphs:
+        taskgraph, label_to_taskid = m(taskgraph, label_to_taskid)
     return taskgraph, label_to_taskid

taskcluster/taskgraph/transforms/job/mozharness.py

@@ -9,6 +9,7 @@ way, and certainly anything using mozharness should use this approach.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
+import slugid
 
 from textwrap import dedent
 
@@ -244,3 +245,27 @@ def mozharness_on_generic_worker(config, job, taskdesc):
         ' '.join(hg_command),
         ' '.join(mh_command)
     ])
+
+
+@run_job_using('buildbot-bridge', 'mozharness', schema=mozharness_run_schema)
+def mozharness_on_buildbot_bridge(config, job, taskdesc):
+    run = job['run']
+    worker = taskdesc['worker']
+    branch = config.params['project']
+    product = run.get('index', {}).get('product', 'firefox')
+
+    worker.pop('env', None)
+
+    worker.update({
+        'buildername': 'OS X 10.7 {} build'.format(branch),
+        'sourcestamp': {
+            'branch': branch,
+            'repository': config.params['head_repository'],
+            'revision': config.params['head_rev'],
+        },
+        'properties': {
+            'product': product,
+            'who': config.params['owner'],
+            'upload_to_task_id': slugid.nice(),
+        }
+    })

taskcluster/taskgraph/transforms/job/mozharness_test.py

@@ -210,17 +210,25 @@ def mozharness_test_on_generic_worker(config, job, taskdesc):
             'path': 'logs/log_warning.log',
             'type': 'file'
         },
-        {
+    ]
+
+    # jittest doesn't have blob_upload_dir
+    if test['test-name'] != 'jittest':
+        artifacts.append({
             'name': 'public/test_info',
             'path': 'build/blobber_upload_dir',
             'type': 'directory'
-        }
-    ]
+        })
 
     build_platform = taskdesc['attributes']['build_platform']
+    build_type = taskdesc['attributes']['build_type']
 
-    target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), build_platform) \
-        if build_platform.startswith('win') else 'target'
+    if build_platform.startswith('win'):
+        target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), build_platform)
+    elif build_platform == 'macosx64' and build_type == 'opt':
+        target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), 'mac')
+    else:
+        target = 'target'
 
     installer_url = get_artifact_url('<build>', mozharness['build-artifact-name'])
 
@@ -330,9 +338,14 @@ def mozharness_test_on_native_engine(config, job, taskdesc):
     mozharness = test['mozharness']
     worker = taskdesc['worker']
 
+    build_platform = taskdesc['attributes']['build_platform']
+    build_type = taskdesc['attributes']['build_type']
+    target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), 'mac') \
+        if build_platform == 'macosx64' and build_type == 'opt' else 'target'
+
     installer_url = get_artifact_url('<build>', mozharness['build-artifact-name'])
     test_packages_url = get_artifact_url('<build>',
-                                         'public/build/target.test_packages.json')
+                                         'public/build/{}.test_packages.json'.format(target))
     mozharness_url = get_artifact_url('<build>',
                                       'public/build/mozharness.zip')
 

taskcluster/taskgraph/transforms/tests.py

@@ -346,7 +346,13 @@ def set_target(config, tests):
     for test in tests:
         build_platform = test['build-platform']
         if build_platform.startswith('macosx'):
-            target = 'target.dmg'
+            if build_platform.split('/')[1] == 'opt':
+                target = 'firefox-{}.en-US.{}.dmg'.format(
+                    get_firefox_version(),
+                    'mac',
+                )
+            else:
+                target = 'target.dmg'
         elif build_platform.startswith('android'):
             if 'geckoview' in test['test-name']:
                 target = 'geckoview_example.apk'

testing/mozharness/mozharness/mozilla/building/buildbase.py

@@ -1419,6 +1419,7 @@ or run without that action (ie: --no-{action})"
             routes.append(template.format(**fmt))
         self.info("Using routes: %s" % routes)
 
+        taskid = self.buildbot_config['properties'].get('upload_to_task_id')
         tc = Taskcluster(
             branch=self.branch,
             rank=pushinfo.pushdate, # Use pushdate as the rank
@@ -1427,10 +1428,13 @@ or run without that action (ie: --no-{action})"
             log_obj=self.log_obj,
             # `upload_to_task_id` is used by mozci to have access to where the artifacts
             # will be uploaded
-            task_id=self.buildbot_config['properties'].get('upload_to_task_id'),
+            task_id=taskid,
         )
 
-        task = tc.create_task(routes)
+        if taskid:
+            task = tc.get_task(taskid)
+        else:
+            task = tc.create_task(routes)
         tc.claim_task(task)
 
         # Only those files uploaded with valid extensions are processed.