Bug 1203552 - Upload buildbot_properties.json for all builds. r=mshal

There are now Buildbot jobs which can be triggered via the Buildbot Bridge.
Buildbot test jobs triggered by BBB builds do not receive the installer and test
urls with a sendchange, hence, failling to run.

In order to pass the installer and test urls to Buildbot test jobs triggered
via BBB (which run with --read-buildbot-configs) we have added functionality
to find these build artifacts by querying TaskCluster.

Up until last quarter this worked because the Buildbot Bridge uploaded a proprties.json
file for every task it created to mirror a Buildbot job (bug 1221091).

Since that is broken, I have found that we can generate a buildbot_properties.json
file and upload it. The current upload mechanism for Buildbot build jobs is that
it creates a TC task and uploads the artifacts there. In this code change, we
determine when a Buildbot job is being driven by a task on TC (aka BBB) and tell
the upload system to use that task instead of creating a new one.

By doing that, tools like Mozilla CI tools can schedule TC graphs out of band by
specifying 'parent_task_id' and getting to the buildbot_properties.json file uploaded
by the parent build job.

--HG--
extra : commitid : 20XrR5whTms
extra : histedit_source : 2f41ffcd33ea7bb0a1fca1c95f92a2c4218b803e%2C7806527c93fc234daf87fa2c704138d4cd368cca

testing/mozharness/mozharness/mozilla/building/buildbase.py

@@ -1232,6 +1232,19 @@ or run without that action (ie: --no-{action})"
                                    testresults,
                                    write_to_file=True)
 
+    def _generate_properties_file(self, path):
+        # TODO it would be better to grab all the properties that were
+        # persisted to file rather than use whats in the buildbot_properties
+        # live object so we become less action dependant.
+        all_current_props = dict(
+            chain(self.buildbot_config['properties'].items(),
+                  self.buildbot_properties.items())
+        )
+        # graph_server_post.py expects a file with 'properties' key
+        graph_props = dict(properties=all_current_props)
+        self.dump_config(path, graph_props)
+
+
     def _graph_server_post(self):
         """graph server post results."""
         self._assert_cfg_valid_for_action(
@@ -1255,18 +1268,8 @@ or run without that action (ie: --no-{action})"
         # graph server takes all our build properties we had initially
         # (buildbot_config) and what we updated to since
         # the script ran (buildbot_properties)
-        # TODO it would be better to grab all the properties that were
-        # persisted to file rather than use whats in the buildbot_properties
-        # live object so we become less action dependant.
-        graph_props_path = os.path.join(c['base_work_dir'],
-                                        "graph_props.json")
-        all_current_props = dict(
-            chain(self.buildbot_config['properties'].items(),
-                  self.buildbot_properties.items())
-        )
-        # graph_server_post.py expects a file with 'properties' key
-        graph_props = dict(properties=all_current_props)
-        self.dump_config(graph_props_path, graph_props)
+        graph_props_path = os.path.join(c['base_work_dir'], "graph_props.json")
+        self._generate_properties_file(graph_props_path)
 
         gs_env = self.query_build_env()
         gs_env.update({'PYTHONPATH': graph_server_path})
@@ -1441,6 +1444,7 @@ or run without that action (ie: --no-{action})"
             self.warning('Skipping S3 file upload: No taskcluster credentials.')
             return
 
+        dirs = self.query_abs_dirs()
         repo = self._query_repo()
         revision = self.query_revision()
         pushinfo = self.vcs_query_pushinfo(repo, revision)
@@ -1466,12 +1470,16 @@ or run without that action (ie: --no-{action})"
             routes.append(template.format(**fmt))
         self.info("Using routes: %s" % routes)
 
-        tc = Taskcluster(self.branch,
-                         pushinfo.pushdate, # Use pushdate as the rank
-                         self.client_id,
-                         self.access_token,
-                         self.log_obj,
-                         )
+        tc = Taskcluster(
+            branch=self.branch,
+            rank=pushinfo.pushdate, # Use pushdate as the rank
+            client_id=self.client_id,
+            access_token=self.access_token,
+            log_obj=self.log_obj,
+            # `upload_to_task_id` is used by mozci to have access to where the artifacts
+            # will be uploaded
+            task_id=self.buildbot_config['properties'].get('upload_to_task_id'),
+        )
 
         # TODO: Bug 1165980 - these should be in tree
         routes.extend([
@@ -1505,6 +1513,17 @@ or run without that action (ie: --no-{action})"
                     if condition(upload_file):
                         self.set_buildbot_property(prop, tc.get_taskcluster_url(upload_file))
                         break
+
+        # Upload a file with all Buildbot properties
+        # This is necessary for Buildbot Bridge test jobs work properly
+        # until we can migrate to TaskCluster
+        properties_path = os.path.join(
+            dirs['base_work_dir'],
+            'buildbot_properties.json'
+        )
+        self._generate_properties_file(properties_path)
+        tc.create_artifact(task, properties_path)
+
         tc.report_completed(task)
 
     def upload_files(self):

testing/mozharness/mozharness/mozilla/taskcluster_helper.py

@@ -13,7 +13,8 @@ class Taskcluster(LogMixin):
     """
     Helper functions to report data to Taskcluster
     """
-    def __init__(self, branch, rank, client_id, access_token, log_obj):
+    def __init__(self, branch, rank, client_id, access_token, log_obj,
+                 task_id=None):
         self.rank = rank
         self.log_obj = log_obj
 
@@ -32,7 +33,7 @@ class Taskcluster(LogMixin):
         taskcluster.config['credentials']['clientId'] = client_id
         taskcluster.config['credentials']['accessToken'] = access_token
         self.taskcluster_queue = taskcluster.Queue()
-        self.task_id = taskcluster.slugId()
+        self.task_id = task_id or taskcluster.slugId()
         self.put_file = taskcluster.utils.putFile
 
     def create_task(self, routes):
@@ -156,18 +157,16 @@ class TaskClusterArtifactFinderMixin(object):
 
         return parent_task_id
 
-    def set_bbb_artifacts(self, task_id):
-        """ Find BBB artifacts through properties.json and set them. """
-        # The tasks which represent a buildbot job only uploads one artifact:
-        # the properties.json file
+    def set_bbb_artifacts(self, task_id, properties_file_path):
+        """ Find BBB artifacts through properties_file_path and set them. """
         p = self.load_json_url(
-            self.url_to_artifact(task_id, 'public/properties.json'))
+            self.url_to_artifact(task_id, properties_file_path))['properties']
 
         # Set importants artifacts for test jobs
         self.set_artifacts(
-            p['packageUrl'][0] if p.get('packageUrl') else None,
-            p['testPackagesUrl'][0] if p.get('testPackagesUrl') else None,
-            p['symbolsUrl'][0] if p.get('symbolsUrl') else None
+            p['packageUrl'] if p.get('packageUrl') else None,
+            p['testPackagesUrl'] if p.get('testPackagesUrl') else None,
+            p['symbolsUrl'] if p.get('symbolsUrl') else None
         )
 
     def set_artifacts(self, installer, tests, symbols):
@@ -197,6 +196,7 @@ class TaskClusterArtifactFinderMixin(object):
         # Task definition
         child_task = self.get_task(child_task_id)
 
+        # Case A: The parent_task_id is defined (mozci scheduling)
         if child_task['payload']['properties'].get('parent_task_id'):
             # parent_task_id is used to point to the task from which to grab artifacts
             # rather than the one we depend on
@@ -205,6 +205,7 @@ class TaskClusterArtifactFinderMixin(object):
             # Find out where the parent task uploaded the build
             parent_task = self.get_task(parent_id)
 
+            # Case 1: The parent task is a pure TC task
             if parent_task['extra'].get('locations'):
                 # Build tasks generated under TC specify where they upload their builds
                 installer_path = parent_task['extra']['locations']['build']
@@ -215,7 +216,18 @@ class TaskClusterArtifactFinderMixin(object):
                     self.url_to_artifact(parent_id, 'public/build/target.crashreporter-symbols.zip')
                 )
             else:
-                self.set_bbb_artifacts(parent_id)
+                # Case 2: The parent task has an associated BBB task
+                # graph_props.json is uploaded in buildbase.py
+                self.set_bbb_artifacts(
+                    task_id=parent_id,
+                    properties_file_path='public/build/buildbot_properties.json'
+                )
+
         else:
+            # Case B: We need to query who the parent is since 'parent_task_id'
+            # was not defined as a Buildbot property
             parent_id = self.find_parent_task_id(child_task_id)
-            self.set_bbb_artifacts(parent_id)
+            self.set_bbb_artifacts(
+                task_id=parent_id,
+                properties_file_path='public/build/buildbot_properties.json'
+            )