diff --git a/testing/mozharness/mozharness/mozilla/building/buildbase.py b/testing/mozharness/mozharness/mozilla/building/buildbase.py index 7a5374f7a155..7fddb74276ad 100755 --- a/testing/mozharness/mozharness/mozilla/building/buildbase.py +++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py @@ -1232,6 +1232,19 @@ or run without that action (ie: --no-{action})" testresults, write_to_file=True) + def _generate_properties_file(self, path): + # TODO it would be better to grab all the properties that were + # persisted to file rather than use whats in the buildbot_properties + # live object so we become less action dependant. + all_current_props = dict( + chain(self.buildbot_config['properties'].items(), + self.buildbot_properties.items()) + ) + # graph_server_post.py expects a file with 'properties' key + graph_props = dict(properties=all_current_props) + self.dump_config(path, graph_props) + + def _graph_server_post(self): """graph server post results.""" self._assert_cfg_valid_for_action( @@ -1255,18 +1268,8 @@ or run without that action (ie: --no-{action})" # graph server takes all our build properties we had initially # (buildbot_config) and what we updated to since # the script ran (buildbot_properties) - # TODO it would be better to grab all the properties that were - # persisted to file rather than use whats in the buildbot_properties - # live object so we become less action dependant. - graph_props_path = os.path.join(c['base_work_dir'], - "graph_props.json") - all_current_props = dict( - chain(self.buildbot_config['properties'].items(), - self.buildbot_properties.items()) - ) - # graph_server_post.py expects a file with 'properties' key - graph_props = dict(properties=all_current_props) - self.dump_config(graph_props_path, graph_props) + graph_props_path = os.path.join(c['base_work_dir'], "graph_props.json") + self._generate_properties_file(graph_props_path) gs_env = self.query_build_env() gs_env.update({'PYTHONPATH': graph_server_path}) @@ -1441,6 +1444,7 @@ or run without that action (ie: --no-{action})" self.warning('Skipping S3 file upload: No taskcluster credentials.') return + dirs = self.query_abs_dirs() repo = self._query_repo() revision = self.query_revision() pushinfo = self.vcs_query_pushinfo(repo, revision) @@ -1466,12 +1470,16 @@ or run without that action (ie: --no-{action})" routes.append(template.format(**fmt)) self.info("Using routes: %s" % routes) - tc = Taskcluster(self.branch, - pushinfo.pushdate, # Use pushdate as the rank - self.client_id, - self.access_token, - self.log_obj, - ) + tc = Taskcluster( + branch=self.branch, + rank=pushinfo.pushdate, # Use pushdate as the rank + client_id=self.client_id, + access_token=self.access_token, + log_obj=self.log_obj, + # `upload_to_task_id` is used by mozci to have access to where the artifacts + # will be uploaded + task_id=self.buildbot_config['properties'].get('upload_to_task_id'), + ) # TODO: Bug 1165980 - these should be in tree routes.extend([ @@ -1505,6 +1513,17 @@ or run without that action (ie: --no-{action})" if condition(upload_file): self.set_buildbot_property(prop, tc.get_taskcluster_url(upload_file)) break + + # Upload a file with all Buildbot properties + # This is necessary for Buildbot Bridge test jobs work properly + # until we can migrate to TaskCluster + properties_path = os.path.join( + dirs['base_work_dir'], + 'buildbot_properties.json' + ) + self._generate_properties_file(properties_path) + tc.create_artifact(task, properties_path) + tc.report_completed(task) def upload_files(self): diff --git a/testing/mozharness/mozharness/mozilla/taskcluster_helper.py b/testing/mozharness/mozharness/mozilla/taskcluster_helper.py index 2f2d75c3de97..30fc3ebb5bb1 100644 --- a/testing/mozharness/mozharness/mozilla/taskcluster_helper.py +++ b/testing/mozharness/mozharness/mozilla/taskcluster_helper.py @@ -13,7 +13,8 @@ class Taskcluster(LogMixin): """ Helper functions to report data to Taskcluster """ - def __init__(self, branch, rank, client_id, access_token, log_obj): + def __init__(self, branch, rank, client_id, access_token, log_obj, + task_id=None): self.rank = rank self.log_obj = log_obj @@ -32,7 +33,7 @@ class Taskcluster(LogMixin): taskcluster.config['credentials']['clientId'] = client_id taskcluster.config['credentials']['accessToken'] = access_token self.taskcluster_queue = taskcluster.Queue() - self.task_id = taskcluster.slugId() + self.task_id = task_id or taskcluster.slugId() self.put_file = taskcluster.utils.putFile def create_task(self, routes): @@ -156,18 +157,16 @@ class TaskClusterArtifactFinderMixin(object): return parent_task_id - def set_bbb_artifacts(self, task_id): - """ Find BBB artifacts through properties.json and set them. """ - # The tasks which represent a buildbot job only uploads one artifact: - # the properties.json file + def set_bbb_artifacts(self, task_id, properties_file_path): + """ Find BBB artifacts through properties_file_path and set them. """ p = self.load_json_url( - self.url_to_artifact(task_id, 'public/properties.json')) + self.url_to_artifact(task_id, properties_file_path))['properties'] # Set importants artifacts for test jobs self.set_artifacts( - p['packageUrl'][0] if p.get('packageUrl') else None, - p['testPackagesUrl'][0] if p.get('testPackagesUrl') else None, - p['symbolsUrl'][0] if p.get('symbolsUrl') else None + p['packageUrl'] if p.get('packageUrl') else None, + p['testPackagesUrl'] if p.get('testPackagesUrl') else None, + p['symbolsUrl'] if p.get('symbolsUrl') else None ) def set_artifacts(self, installer, tests, symbols): @@ -197,6 +196,7 @@ class TaskClusterArtifactFinderMixin(object): # Task definition child_task = self.get_task(child_task_id) + # Case A: The parent_task_id is defined (mozci scheduling) if child_task['payload']['properties'].get('parent_task_id'): # parent_task_id is used to point to the task from which to grab artifacts # rather than the one we depend on @@ -205,6 +205,7 @@ class TaskClusterArtifactFinderMixin(object): # Find out where the parent task uploaded the build parent_task = self.get_task(parent_id) + # Case 1: The parent task is a pure TC task if parent_task['extra'].get('locations'): # Build tasks generated under TC specify where they upload their builds installer_path = parent_task['extra']['locations']['build'] @@ -215,7 +216,18 @@ class TaskClusterArtifactFinderMixin(object): self.url_to_artifact(parent_id, 'public/build/target.crashreporter-symbols.zip') ) else: - self.set_bbb_artifacts(parent_id) + # Case 2: The parent task has an associated BBB task + # graph_props.json is uploaded in buildbase.py + self.set_bbb_artifacts( + task_id=parent_id, + properties_file_path='public/build/buildbot_properties.json' + ) + else: + # Case B: We need to query who the parent is since 'parent_task_id' + # was not defined as a Buildbot property parent_id = self.find_parent_task_id(child_task_id) - self.set_bbb_artifacts(parent_id) + self.set_bbb_artifacts( + task_id=parent_id, + properties_file_path='public/build/buildbot_properties.json' + )