From 80bb005dadc2a3395992888d4b7bc2f36434b59e Mon Sep 17 00:00:00 2001 From: Cameron Dawson Date: Tue, 6 Aug 2013 17:01:03 -0700 Subject: [PATCH 1/4] add a couple filters to the result_set list and singular --- treeherder/model/derived/jobs.py | 14 +++++++++-- treeherder/model/sql/jobs.json | 21 ++++++++++++----- treeherder/webapp/api/views.py | 40 +++++++++++++++++++++++++------- 3 files changed, 59 insertions(+), 16 deletions(-) diff --git a/treeherder/model/derived/jobs.py b/treeherder/model/derived/jobs.py index dc19479c4..98f1bf055 100644 --- a/treeherder/model/derived/jobs.py +++ b/treeherder/model/derived/jobs.py @@ -131,35 +131,45 @@ class JobsModel(TreeherderModelBase): return id_iter.get_column_data('id') - def get_result_set_list(self, page, limit): + def get_result_set_list(self, page, limit, **kwargs): """ Retrieve a list of ``result_sets`` (also known as ``pushes``) with associated revisions. No jobs Mainly used by the restful api to list the pushes in the UI """ + repl = [""] + if "pusher" in kwargs: + repl = [" WHERE `rev`.`author` = '{0}'".format(kwargs["pusher"])] + proc = "jobs.selects.get_result_set_list" push_dict = self.get_jobs_dhub().execute( proc=proc, placeholders=[page, limit], debug_show=self.DEBUG, return_type='iter', + replace=repl, ) return push_dict - def get_result_set_job_list(self, result_set_id): + def get_result_set_job_list(self, result_set_id, **kwargs): """ Retrieve a list of ``jobs`` and results for a result_set. Mainly used by the restful api to list the job results in the UI """ + repl = [""] + if "job_name" in kwargs: + repl = [" AND jt.`name` = '{0}'".format(kwargs["job_name"])] + proc = "jobs.selects.get_result_set_job_list" push_dict = self.get_jobs_dhub().execute( proc=proc, placeholders=[result_set_id], debug_show=self.DEBUG, return_type='iter', + replace=repl, ) return push_dict diff --git a/treeherder/model/sql/jobs.json b/treeherder/model/sql/jobs.json index 6a288ab3e..0f880ebb0 100644 --- a/treeherder/model/sql/jobs.json +++ b/treeherder/model/sql/jobs.json @@ -177,7 +177,9 @@ ON rs.id = rm.result_set_id LEFT JOIN revision as rev ON rm.revision_id = rev.id - LIMIT ?,?", + REP0 + LIMIT ?,? + ", "host": "read_host" }, "get_result_set_by_id":{ @@ -202,10 +204,13 @@ j.`job_guid`, j.`build_platform_id`, mp.`platform`, - m.`name`, - jt.`name`, - jt.`symbol`, - jt.`description`, + m.`name` as machine_name, + jt.`name` as jt_name, + jt.`symbol` as jt_symbol, + jt.`description` as jt_description, + jg.`name` as jg_name, + jg.`symbol` as jg_symbol, + jg.`description` as jg_description, j.`who`, j.`result_set_id`, j.`result`, @@ -219,7 +224,11 @@ ON j.`build_platform_id` = bp.id LEFT JOIN `treeherder`.`job_type` as jt ON j.`job_type_id` = jt.id - WHERE `result_set_id` = ?", + LEFT JOIN `treeherder`.`job_group` as jg + ON jt.`job_group_id` = jg.id + WHERE `result_set_id` = ? + REP0 + ", "host": "read_host" } } diff --git a/treeherder/webapp/api/views.py b/treeherder/webapp/api/views.py index b2cfd2fbd..7e30f28d5 100644 --- a/treeherder/webapp/api/views.py +++ b/treeherder/webapp/api/views.py @@ -146,9 +146,14 @@ class ResultSetViewSet(viewsets.ViewSet): """ try: page = request.QUERY_PARAMS.get('page', 0) + pusher = request.QUERY_PARAMS.get('pusher', None) + kwargs = {} + if pusher: + kwargs["pusher"] = pusher + jm = JobsModel(project) - objs = jm.get_result_set_list(page, 1000) + objs = jm.get_result_set_list(page, 1000, **kwargs) return Response(objs) except DatasetNotFoundError as e: return Response({"message": unicode(e)}, status=404) @@ -163,14 +168,20 @@ class ResultSetViewSet(viewsets.ViewSet): A color-based warning level based on the most severe level in the list of jobs. + + @@@ - This needs a better way. """ job_states = set([x["result"] for x in jobs]) if "busted" in job_states: return "red" + if "fail" in job_states: + return "red" elif "orange" in job_states: return "orange" elif "pending" in job_states: return "grey" + elif "retry" in job_states: + return "grey" elif "running" in job_states: return "grey" else: @@ -180,21 +191,34 @@ class ResultSetViewSet(viewsets.ViewSet): """ GET method implementation for detail view of ``resultset`` """ + job_name = request.QUERY_PARAMS.get('job_name', None) + kwargs = {} + if job_name: + kwargs["job_name"] = job_name + try: jm = JobsModel(project) rs = list(jm.get_result_set_by_id(pk))[0] - jobs_ungrouped = list(jm.get_result_set_job_list(pk)) + jobs_ungrouped = list(jm.get_result_set_job_list(pk, **kwargs)) # group these by their platforms for return jobs_sorted = sorted(jobs_ungrouped, key=lambda x: x["platform"]) import itertools - rs["jobs"] = [] + rs["platforms"] = [] + # job_groups by platform for k, g in itertools.groupby(jobs_sorted, key=lambda x: x["platform"]): - jobs = list(g) - rs["jobs"].append({ + job_groups = sorted(list(g), key=lambda x: x["jg_symbol"]) + platform = { "platform": k, - "warning_level": self.get_warning_level(jobs), - "jobs": jobs - }) + } + rs["platforms"].append(platform) + platform["groups"] = [] + for jg_k, jg_g in itertools.groupby(job_groups, key=lambda x: x["jg_symbol"]): + jobs = list(jg_g) + platform["groups"].append({ + "symbol": jg_k, + "warning_level": self.get_warning_level(jobs), + "jobs": jobs + }) return Response(rs) except DatasetNotFoundError as e: return Response( From 5b362cb4bcb48fbc60cc0651e9863d00678ba8bb Mon Sep 17 00:00:00 2001 From: Cameron Dawson Date: Tue, 6 Aug 2013 17:26:12 -0700 Subject: [PATCH 2/4] better filtering technique --- treeherder/model/derived/jobs.py | 8 ++++---- treeherder/model/sql/jobs.json | 13 +++++++------ treeherder/webapp/api/views.py | 14 +++++++------- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/treeherder/model/derived/jobs.py b/treeherder/model/derived/jobs.py index 98f1bf055..13fb4a326 100644 --- a/treeherder/model/derived/jobs.py +++ b/treeherder/model/derived/jobs.py @@ -139,8 +139,8 @@ class JobsModel(TreeherderModelBase): Mainly used by the restful api to list the pushes in the UI """ repl = [""] - if "pusher" in kwargs: - repl = [" WHERE `rev`.`author` = '{0}'".format(kwargs["pusher"])] + if "author" in kwargs: + repl = [" AND `rev`.`author` = '{0}'".format(kwargs["author"])] proc = "jobs.selects.get_result_set_list" push_dict = self.get_jobs_dhub().execute( @@ -160,8 +160,8 @@ class JobsModel(TreeherderModelBase): Mainly used by the restful api to list the job results in the UI """ repl = [""] - if "job_name" in kwargs: - repl = [" AND jt.`name` = '{0}'".format(kwargs["job_name"])] + if "job_type_name" in kwargs: + repl = [" AND jt.`name` = '{0}'".format(kwargs["job_type_name"])] proc = "jobs.selects.get_result_set_job_list" push_dict = self.get_jobs_dhub().execute( diff --git a/treeherder/model/sql/jobs.json b/treeherder/model/sql/jobs.json index 0f880ebb0..ae7b4eae7 100644 --- a/treeherder/model/sql/jobs.json +++ b/treeherder/model/sql/jobs.json @@ -177,6 +177,7 @@ ON rs.id = rm.result_set_id LEFT JOIN revision as rev ON rm.revision_id = rev.id + WHERE 1 REP0 LIMIT ?,? ", @@ -205,12 +206,12 @@ j.`build_platform_id`, mp.`platform`, m.`name` as machine_name, - jt.`name` as jt_name, - jt.`symbol` as jt_symbol, - jt.`description` as jt_description, - jg.`name` as jg_name, - jg.`symbol` as jg_symbol, - jg.`description` as jg_description, + jt.`name` as job_type_name, + jt.`symbol` as job_type_symbol, + jt.`description` as job_type_description, + jg.`name` as job_group_name, + jg.`symbol` as job_group_symbol, + jg.`description` as job_group_description, j.`who`, j.`result_set_id`, j.`result`, diff --git a/treeherder/webapp/api/views.py b/treeherder/webapp/api/views.py index 7e30f28d5..e1951c891 100644 --- a/treeherder/webapp/api/views.py +++ b/treeherder/webapp/api/views.py @@ -191,28 +191,28 @@ class ResultSetViewSet(viewsets.ViewSet): """ GET method implementation for detail view of ``resultset`` """ - job_name = request.QUERY_PARAMS.get('job_name', None) - kwargs = {} - if job_name: - kwargs["job_name"] = job_name + filters = ["job_type_name"] try: jm = JobsModel(project) rs = list(jm.get_result_set_by_id(pk))[0] - jobs_ungrouped = list(jm.get_result_set_job_list(pk, **kwargs)) + jobs_ungrouped = list(jm.get_result_set_job_list( + pk, + **dict((k, v) for k, v in request.QUERY_PARAMS.iteritems() if k in filters) + )) # group these by their platforms for return jobs_sorted = sorted(jobs_ungrouped, key=lambda x: x["platform"]) import itertools rs["platforms"] = [] # job_groups by platform for k, g in itertools.groupby(jobs_sorted, key=lambda x: x["platform"]): - job_groups = sorted(list(g), key=lambda x: x["jg_symbol"]) + job_groups = sorted(list(g), key=lambda x: x["job_group_symbol"]) platform = { "platform": k, } rs["platforms"].append(platform) platform["groups"] = [] - for jg_k, jg_g in itertools.groupby(job_groups, key=lambda x: x["jg_symbol"]): + for jg_k, jg_g in itertools.groupby(job_groups, key=lambda x: x["job_group_symbol"]): jobs = list(jg_g) platform["groups"].append({ "symbol": jg_k, From 907508092df4c81432ed555fe55d7a527b52d842 Mon Sep 17 00:00:00 2001 From: Cameron Dawson Date: Tue, 6 Aug 2013 17:30:17 -0700 Subject: [PATCH 3/4] code cleanup --- treeherder/webapp/api/views.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/treeherder/webapp/api/views.py b/treeherder/webapp/api/views.py index e1951c891..f2318c2a3 100644 --- a/treeherder/webapp/api/views.py +++ b/treeherder/webapp/api/views.py @@ -144,16 +144,18 @@ class ResultSetViewSet(viewsets.ViewSet): """ GET method for list of ``resultset`` records with revisions """ + + filters = ["author"] + try: page = request.QUERY_PARAMS.get('page', 0) - pusher = request.QUERY_PARAMS.get('pusher', None) - kwargs = {} - if pusher: - kwargs["pusher"] = pusher - jm = JobsModel(project) - objs = jm.get_result_set_list(page, 1000, **kwargs) + objs = jm.get_result_set_list( + page, + 1000, + **dict((k, v) for k, v in request.QUERY_PARAMS.iteritems() if k in filters) + ) return Response(objs) except DatasetNotFoundError as e: return Response({"message": unicode(e)}, status=404) From d5de5cdf8f839df29b03a14557f2ecfd22bfcc66 Mon Sep 17 00:00:00 2001 From: Cameron Dawson Date: Wed, 7 Aug 2013 13:07:05 -0700 Subject: [PATCH 4/4] filter updates --- treeherder/webapp/api/views.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/treeherder/webapp/api/views.py b/treeherder/webapp/api/views.py index f2318c2a3..022bffd53 100644 --- a/treeherder/webapp/api/views.py +++ b/treeherder/webapp/api/views.py @@ -164,7 +164,7 @@ class ResultSetViewSet(viewsets.ViewSet): finally: jm.disconnect() - def get_warning_level(self, jobs): + def get_warning_level(self, groups): """ Return the most severe warning level for a list of jobs. @@ -173,7 +173,11 @@ class ResultSetViewSet(viewsets.ViewSet): @@@ - This needs a better way. """ - job_states = set([x["result"] for x in jobs]) + job_states = [] + for group in groups: + job_states.extend([job["result"] for job in group["jobs"]]) + + job_states = set(job_states) if "busted" in job_states: return "red" if "fail" in job_states: @@ -209,18 +213,19 @@ class ResultSetViewSet(viewsets.ViewSet): # job_groups by platform for k, g in itertools.groupby(jobs_sorted, key=lambda x: x["platform"]): job_groups = sorted(list(g), key=lambda x: x["job_group_symbol"]) - platform = { - "platform": k, - } - rs["platforms"].append(platform) - platform["groups"] = [] + groups = [] for jg_k, jg_g in itertools.groupby(job_groups, key=lambda x: x["job_group_symbol"]): jobs = list(jg_g) - platform["groups"].append({ + groups.append({ "symbol": jg_k, - "warning_level": self.get_warning_level(jobs), "jobs": jobs }) + rs["platforms"].append({ + "name": k, + "groups": groups, + "warning_level": self.get_warning_level(groups) + }) + return Response(rs) except DatasetNotFoundError as e: return Response(