From bde96f1fd7855b34be63172bea73d9a77d314ce3 Mon Sep 17 00:00:00 2001
From: Cameron Dawson <cdawson@mozilla.com>
Date: Tue, 28 Jan 2014 17:01:29 -0800
Subject: [PATCH 01/11] add ``full`` param to get full data.  Set to false for
 less.

---
 treeherder/model/derived/jobs.py | 52 +++++++++++++++++++------------
 treeherder/model/sql/jobs.json   | 53 ++++++++++++++++++++++++++++++++
 treeherder/webapp/api/views.py   | 33 ++++++++++++--------
 3 files changed, 105 insertions(+), 33 deletions(-)

diff --git a/treeherder/model/derived/jobs.py b/treeherder/model/derived/jobs.py
index e95429f69..5bcd3fa26 100644
--- a/treeherder/model/derived/jobs.py
+++ b/treeherder/model/derived/jobs.py
@@ -75,12 +75,14 @@ class JobsModel(TreeherderModelBase):
         )
         return data
 
-    def get_job_list(self, offset, limit, **kwargs):
+    def get_job_list(self, offset, limit, full=True, **kwargs):
         """
         Retrieve a list of jobs.
         Mainly used by the restful api to list the jobs
 
         joblist: a list of job ids to limit which jobs are returned.
+        full: whether to return all reference data or just what is
+              needed for the UI.
         """
         filter_str = ""
 
@@ -89,7 +91,10 @@ class JobsModel(TreeherderModelBase):
 
         repl = [self.refdata_model.get_db_name(), filter_str]
 
-        proc = "jobs.selects.get_job_list"
+        if full:
+            proc = "jobs.selects.get_job_list_full"
+        else:
+            proc = "jobs.selects.get_job_list"
         data = self.get_jobs_dhub().execute(
             proc=proc,
             replace=repl,
@@ -201,10 +206,11 @@ class JobsModel(TreeherderModelBase):
 
         return result_set_id_lookup
 
-    def get_result_set_list(self, offset, limit, **kwargs):
+    def get_result_set_list(self, offset, limit, full, **kwargs):
         """
         Retrieve a list of ``result_sets`` (also known as ``pushes``)
-        with associated revisions.  No jobs
+        If ``full`` is set to ``True`` then return revisions, too.
+        No jobs
 
         Mainly used by the restful api to list the pushes in the UI
         """
@@ -244,20 +250,23 @@ class JobsModel(TreeherderModelBase):
         return_list = []
         for result in result_set_ids:
 
-            detail = aggregate_details[ result['id'] ][0]
-
-            return_list.append(
-                {
-                    "id":result['id'],
-                    "revision_hash":result['revision_hash'],
-                    "push_timestamp":result['push_timestamp'],
-                    "repository_id":detail['repository_id'],
-                    "revision":detail['revision'],
-                    "author":detail['author'],
-                    "comments":detail['comments'],
-                    "revision_list":aggregate_details[ result['id'] ]
-                    }
-                )
+            detail = aggregate_details[result['id']][0]
+            list_item = {
+                "id": result['id'],
+                "revision_hash": result['revision_hash'],
+                "push_timestamp": result['push_timestamp'],
+                "repository_id": detail['repository_id'],
+                "revision": detail['revision'],
+                "author": detail['author'],
+                "revision_count": len(aggregate_details[result['id']])
+            }
+            if full:
+                list_item.update({
+                    "full": full,
+                    "comments": detail['comments'],
+                    "revision_list": aggregate_details[result['id']]
+                })
+            return_list.append(list_item)
 
         return return_list
 
@@ -330,7 +339,7 @@ class JobsModel(TreeherderModelBase):
 
         return aggregate_details
 
-    def get_result_set_job_list(self, result_set_ids, **kwargs):
+    def get_result_set_job_list(self, result_set_ids, full=True, **kwargs):
         """
         Retrieve a list of ``jobs`` and results for a result_set.
 
@@ -352,7 +361,10 @@ class JobsModel(TreeherderModelBase):
         if "job_type_name" in kwargs:
             repl.append(" AND jt.`name` = '{0}'".format(kwargs["job_type_name"]))
 
-        proc = "jobs.selects.get_result_set_job_list"
+        if full:
+            proc = "jobs.selects.get_result_set_job_list_full"
+        else:
+            proc = "jobs.selects.get_result_set_job_list"
         data = self.get_jobs_dhub().execute(
             proc=proc,
             placeholders=result_set_ids,
diff --git a/treeherder/model/sql/jobs.json b/treeherder/model/sql/jobs.json
index 6d88d1e5f..2e94a3c9d 100644
--- a/treeherder/model/sql/jobs.json
+++ b/treeherder/model/sql/jobs.json
@@ -201,6 +201,31 @@
             "host":"read_host"
         },
         "get_job_list":{
+            "sql":"SELECT
+                    j.id,
+                    j.`option_collection_hash`,
+                    mp.`platform` as platform,
+                    jt.`name` as job_type_name,
+                    jt.`symbol` as job_type_symbol,
+					jg.`name` as job_group_name,
+                    jg.`symbol` as job_group_symbol,
+                    j.`result_set_id`,
+                    j.`result`,
+                    j.`state`
+                  FROM `job` as j
+                  LEFT JOIN `REP0`.`machine_platform` as mp
+                    ON j.`machine_platform_id` = mp.id
+                  LEFT JOIN `REP0`.`job_type` as jt
+                    ON j.`job_type_id` = jt.id
+				  LEFT JOIN `REP0`.`job_group` as jg
+                    ON jt.`job_group_id` = jg.id
+                  WHERE 1
+                  REP1
+                  LIMIT ?,?",
+
+            "host":"read_host"
+        },
+        "get_job_list_full":{
             "sql":"SELECT
                     j.id,
                     j.`job_guid`,
@@ -382,6 +407,34 @@
             "host": "read_host"
         },
         "get_result_set_job_list":{
+            "sql":"SELECT
+                    j.`id`,
+                    j.`option_collection_hash`,
+                    mp.`platform` as platform,
+                    jt.`name` as job_type_name,
+                    jt.`symbol` as job_type_symbol,
+					jg.`name` as job_group_name,
+                    jg.`symbol` as job_group_symbol,
+                    j.`result_set_id`,
+                    j.`id` as job_id,
+                    j.`result`,
+                    j.`state`,
+                    j.`reason`
+                  FROM `job` as j
+                  LEFT JOIN `REP0`.`machine` as m
+                    ON j.`machine_id` = m.id
+                  LEFT JOIN `REP0`.`machine_platform` as mp
+                    ON j.`machine_platform_id` = mp.id
+                  LEFT JOIN `REP0`.`job_type` as jt
+                    ON j.`job_type_id` = jt.id
+				  LEFT JOIN `REP0`.`job_group` as jg
+                    ON jt.`job_group_id` = jg.id
+                  WHERE j.`result_set_id` IN (REP1)
+                  REP2
+                  ",
+            "host": "read_host"
+        },
+        "get_result_set_job_list_full":{
             "sql":"SELECT
                     j.`job_guid`,
                     j.`job_coalesced_to_guid`,
diff --git a/treeherder/webapp/api/views.py b/treeherder/webapp/api/views.py
index 3a7c7e4ab..9119f8aff 100644
--- a/treeherder/webapp/api/views.py
+++ b/treeherder/webapp/api/views.py
@@ -1,6 +1,8 @@
 import simplejson as json
 import itertools
 
+from collections import Counter
+
 from django.conf import settings
 from rest_framework import viewsets
 from rest_framework.response import Response
@@ -208,10 +210,12 @@ class JobsViewSet(viewsets.ViewSet):
 
         offset = int(request.QUERY_PARAMS.get('offset', 0))
         count = int(request.QUERY_PARAMS.get('count', 10))
+        full = bool(request.QUERY_PARAMS.get('full', True))
 
         objs = jm.get_job_list(
             offset,
             count,
+            full,
             **dict((k, v) for k, v in request.QUERY_PARAMS.iteritems()
                    if k in filters)
         )
@@ -284,14 +288,16 @@ class ResultSetViewSet(viewsets.ViewSet):
 
         offset = int(request.QUERY_PARAMS.get('offset', 0))
         count = int(request.QUERY_PARAMS.get('count', 10))
-
+        full = request.QUERY_PARAMS.get('full', "True").lower() == "true"
+        # full=False
         objs = jm.get_result_set_list(
             offset,
             count,
+            full,
             **dict((k, v) for k, v in request.QUERY_PARAMS.iteritems()
                    if k in filters)
         )
-        return Response(self.get_resultsets_with_jobs(jm, objs, {}))
+        return Response(self.get_resultsets_with_jobs(jm, objs, full, {}))
 
     @with_jobs
     def retrieve(self, request, project, jm, pk=None):
@@ -299,6 +305,9 @@ class ResultSetViewSet(viewsets.ViewSet):
         GET method implementation for detail view of ``resultset``
         """
         filters = ["job_type_name"]
+
+        full = bool(request.QUERY_PARAMS.get('full', True))
+
         filter_kwargs = dict(
             (k, v) for k, v in request.QUERY_PARAMS.iteritems()
             if k in filters
@@ -306,13 +315,13 @@ class ResultSetViewSet(viewsets.ViewSet):
 
         rs = jm.get_result_set_by_id(pk)
         if rs:
-            resultsets = self.get_resultsets_with_jobs(jm, [rs[0]], filter_kwargs)
+            resultsets = self.get_resultsets_with_jobs(jm, [rs[0]], full, filter_kwargs)
             return Response(resultsets[0])
         else:
             return Response("No resultset with id: {0}".format(pk), 404)
 
     @staticmethod
-    def get_resultsets_with_jobs(jm, rs_list, filter_kwargs):
+    def get_resultsets_with_jobs(jm, rs_list, full, filter_kwargs):
         """Convert db result of resultsets in a list to JSON"""
 
         # Fetch the job results all at once, then parse them out in memory.
@@ -323,6 +332,7 @@ class ResultSetViewSet(viewsets.ViewSet):
 
         jobs_ungrouped = jm.get_result_set_job_list(
             rs_map.keys(),
+            full,
             **filter_kwargs
         )
 
@@ -352,8 +362,7 @@ class ResultSetViewSet(viewsets.ViewSet):
             # of resultsets to be returned.
             del(rs_map[rs_id])
 
-            result_types = []
-            job_count = 0
+            job_counts = Counter()
 
             #itertools needs the elements to be sorted by the grouper
             by_platform = sorted(list(resultset_group), key=platform_grouper)
@@ -391,10 +400,10 @@ class ResultSetViewSet(viewsets.ViewSet):
                             kwargs={"project": jm.project, "pk": job["id"]})
 
                         if job["state"] == "completed":
-                            result_types.append(job["result"])
+                            job_counts[job["result"]] += 1
                         else:
-                            result_types.append(job["state"])
-                        job_count += 1
+                            job_counts[job["state"]] += 1
+                        job_counts["total"] += 1
 
                 platforms.append({
                     "name": platform_name,
@@ -406,8 +415,7 @@ class ResultSetViewSet(viewsets.ViewSet):
             #can be used to determine the resultset's severity
             resultset.update({
                 "platforms": platforms,
-                "result_types": list(set(result_types)),
-                "job_count": job_count,
+                "job_counts": job_counts,
             })
 
         # the resultsets left in the map have no jobs, so fill in the fields
@@ -415,8 +423,7 @@ class ResultSetViewSet(viewsets.ViewSet):
         for rs in rs_map.values():
             rs.update({
                 "platforms": [],
-                "result_types": [],
-                "job_count": 0,
+                "job_counts": {"total": 0},
             })
             resultsets.append(rs)
         return sorted(resultsets, key=lambda x: x["push_timestamp"], reverse=True)

From 5e63dc1fd835870964155df7978b39626844aad6 Mon Sep 17 00:00:00 2001
From: Cameron Dawson <cdawson@mozilla.com>
Date: Tue, 28 Jan 2014 17:01:29 -0800
Subject: [PATCH 02/11] add ``full`` param to get full data.  Set to false for
 less.

---
 treeherder/model/derived/jobs.py | 53 +++++++++++++++++++-------------
 treeherder/model/sql/jobs.json   | 53 ++++++++++++++++++++++++++++++++
 treeherder/webapp/api/views.py   | 40 ++++++++++++++++--------
 3 files changed, 113 insertions(+), 33 deletions(-)

diff --git a/treeherder/model/derived/jobs.py b/treeherder/model/derived/jobs.py
index 10ffeb384..356d6fbe4 100644
--- a/treeherder/model/derived/jobs.py
+++ b/treeherder/model/derived/jobs.py
@@ -103,12 +103,14 @@ class JobsModel(TreeherderModelBase):
         )
         return data
 
-    def get_job_list(self, offset, limit, conditions=None):
+    def get_job_list(self, offset, limit, full=True, conditions=None):
         """
         Retrieve a list of jobs.
         Mainly used by the restful api to list the jobs
 
         joblist: a list of job ids to limit which jobs are returned.
+        full: whether to return all reference data or just what is
+              needed for the UI.
         """
 
         placeholders = []
@@ -131,8 +133,10 @@ class JobsModel(TreeherderModelBase):
 
         repl = [self.refdata_model.get_db_name(), replace_str]
 
-        proc = "jobs.selects.get_job_list"
-
+        if full:
+            proc = "jobs.selects.get_job_list_full"
+        else:
+            proc = "jobs.selects.get_job_list"
         data = self.get_jobs_dhub().execute(
             proc=proc,
             replace=repl,
@@ -254,10 +258,11 @@ class JobsModel(TreeherderModelBase):
 
         return result_set_id_lookup
 
-    def get_result_set_list(self, offset, limit, conditions=None):
+    def get_result_set_list(self, offset, limit, full=True, conditions=None):
         """
         Retrieve a list of ``result_sets`` (also known as ``pushes``)
-        with associated revisions.  No jobs
+        If ``full`` is set to ``True`` then return revisions, too.
+        No jobs
 
         Mainly used by the restful api to list the pushes in the UI
         """
@@ -301,20 +306,23 @@ class JobsModel(TreeherderModelBase):
         return_list = []
         for result in result_set_ids:
 
-            detail = aggregate_details[ result['id'] ][0]
-
-            return_list.append(
-                {
-                    "id":result['id'],
-                    "revision_hash":result['revision_hash'],
-                    "push_timestamp":result['push_timestamp'],
-                    "repository_id":detail['repository_id'],
-                    "revision":detail['revision'],
-                    "author":detail['author'],
-                    "comments":detail['comments'],
-                    "revision_list":aggregate_details[ result['id'] ]
-                    }
-                )
+            detail = aggregate_details[result['id']][0]
+            list_item = {
+                "id": result['id'],
+                "revision_hash": result['revision_hash'],
+                "push_timestamp": result['push_timestamp'],
+                "repository_id": detail['repository_id'],
+                "revision": detail['revision'],
+                "author": detail['author'],
+                "revision_count": len(aggregate_details[result['id']])
+            }
+            if full:
+                list_item.update({
+                    "full": full,
+                    "comments": detail['comments'],
+                    "revision_list": aggregate_details[result['id']]
+                })
+            return_list.append(list_item)
 
         return return_list
 
@@ -387,7 +395,7 @@ class JobsModel(TreeherderModelBase):
 
         return aggregate_details
 
-    def get_result_set_job_list(self, result_set_ids, **kwargs):
+    def get_result_set_job_list(self, result_set_ids, full=True, **kwargs):
         """
         Retrieve a list of ``jobs`` and results for a result_set.
 
@@ -409,7 +417,10 @@ class JobsModel(TreeherderModelBase):
         if "job_type_name" in kwargs:
             repl.append(" AND jt.`name` = '{0}'".format(kwargs["job_type_name"]))
 
-        proc = "jobs.selects.get_result_set_job_list"
+        if full:
+            proc = "jobs.selects.get_result_set_job_list_full"
+        else:
+            proc = "jobs.selects.get_result_set_job_list"
         data = self.get_jobs_dhub().execute(
             proc=proc,
             placeholders=result_set_ids,
diff --git a/treeherder/model/sql/jobs.json b/treeherder/model/sql/jobs.json
index 76c3923b1..768990e86 100644
--- a/treeherder/model/sql/jobs.json
+++ b/treeherder/model/sql/jobs.json
@@ -208,6 +208,31 @@
             "host":"read_host"
         },
         "get_job_list":{
+            "sql":"SELECT
+                    j.id,
+                    j.`option_collection_hash`,
+                    mp.`platform` as platform,
+                    jt.`name` as job_type_name,
+                    jt.`symbol` as job_type_symbol,
+					jg.`name` as job_group_name,
+                    jg.`symbol` as job_group_symbol,
+                    j.`result_set_id`,
+                    j.`result`,
+                    j.`state`
+                  FROM `job` as j
+                  LEFT JOIN `REP0`.`machine_platform` as mp
+                    ON j.`machine_platform_id` = mp.id
+                  LEFT JOIN `REP0`.`job_type` as jt
+                    ON j.`job_type_id` = jt.id
+				  LEFT JOIN `REP0`.`job_group` as jg
+                    ON jt.`job_group_id` = jg.id
+                  WHERE 1
+                  REP1
+                  LIMIT ?,?",
+
+            "host":"read_host"
+        },
+        "get_job_list_full":{
             "sql":"SELECT
                     j.id,
                     j.`job_guid`,
@@ -387,6 +412,34 @@
             "host": "read_host"
         },
         "get_result_set_job_list":{
+            "sql":"SELECT
+                    j.`id`,
+                    j.`option_collection_hash`,
+                    mp.`platform` as platform,
+                    jt.`name` as job_type_name,
+                    jt.`symbol` as job_type_symbol,
+					jg.`name` as job_group_name,
+                    jg.`symbol` as job_group_symbol,
+                    j.`result_set_id`,
+                    j.`id` as job_id,
+                    j.`result`,
+                    j.`state`,
+                    j.`reason`
+                  FROM `job` as j
+                  LEFT JOIN `REP0`.`machine` as m
+                    ON j.`machine_id` = m.id
+                  LEFT JOIN `REP0`.`machine_platform` as mp
+                    ON j.`machine_platform_id` = mp.id
+                  LEFT JOIN `REP0`.`job_type` as jt
+                    ON j.`job_type_id` = jt.id
+				  LEFT JOIN `REP0`.`job_group` as jg
+                    ON jt.`job_group_id` = jg.id
+                  WHERE j.`result_set_id` IN (REP1)
+                  REP2
+                  ",
+            "host": "read_host"
+        },
+        "get_result_set_job_list_full":{
             "sql":"SELECT
                     j.`job_guid`,
                     j.`job_coalesced_to_guid`,
diff --git a/treeherder/webapp/api/views.py b/treeherder/webapp/api/views.py
index ec019165b..60c071130 100644
--- a/treeherder/webapp/api/views.py
+++ b/treeherder/webapp/api/views.py
@@ -1,6 +1,8 @@
 import simplejson as json
 import itertools
 
+from collections import Counter
+
 from django.conf import settings
 from rest_framework import viewsets
 from rest_framework.response import Response
@@ -210,6 +212,17 @@ class JobsViewSet(viewsets.ViewSet):
         offset, limit = limit_condition[1].split(",")
         objs = jm.get_job_list(offset, limit, filters)
 
+        offset = int(request.QUERY_PARAMS.get('offset', 0))
+        count = int(request.QUERY_PARAMS.get('count', 10))
+        full = request.QUERY_PARAMS.get('full', 'true').lower() == 'true'
+
+        objs = jm.get_job_list(
+            offset,
+            count,
+            full,
+            **dict((k, v) for k, v in request.QUERY_PARAMS.iteritems()
+                   if k in filters)
+        )
         if objs:
             option_collections = jm.refdata_model.get_all_option_collections()
             for job in objs:
@@ -278,13 +291,15 @@ class ResultSetViewSet(viewsets.ViewSet):
 
         limit_condition = filters.pop("limit", set([("=", "0,10")])).pop()
         offset, limit = limit_condition[1].split(",")
+        full = request.QUERY_PARAMS.get('full', "true").lower() == "true"
 
         objs = jm.get_result_set_list(
             offset,
             limit,
+            full,
             filters
         )
-        return Response(self.get_resultsets_with_jobs(jm, objs, {}))
+        return Response(self.get_resultsets_with_jobs(jm, objs, full, {}))
 
     @with_jobs
     def retrieve(self, request, project, jm, pk=None):
@@ -292,6 +307,9 @@ class ResultSetViewSet(viewsets.ViewSet):
         GET method implementation for detail view of ``resultset``
         """
         filters = ["job_type_name"]
+
+        full = request.QUERY_PARAMS.get('full', "true").lower() == "true"
+
         filter_kwargs = dict(
             (k, v) for k, v in request.QUERY_PARAMS.iteritems()
             if k in filters
@@ -299,13 +317,13 @@ class ResultSetViewSet(viewsets.ViewSet):
 
         rs = jm.get_result_set_by_id(pk)
         if rs:
-            resultsets = self.get_resultsets_with_jobs(jm, [rs[0]], filter_kwargs)
+            resultsets = self.get_resultsets_with_jobs(jm, [rs[0]], full, filter_kwargs)
             return Response(resultsets[0])
         else:
             return Response("No resultset with id: {0}".format(pk), 404)
 
     @staticmethod
-    def get_resultsets_with_jobs(jm, rs_list, filter_kwargs):
+    def get_resultsets_with_jobs(jm, rs_list, full, filter_kwargs):
         """Convert db result of resultsets in a list to JSON"""
 
         # Fetch the job results all at once, then parse them out in memory.
@@ -316,6 +334,7 @@ class ResultSetViewSet(viewsets.ViewSet):
 
         jobs_ungrouped = jm.get_result_set_job_list(
             rs_map.keys(),
+            full,
             **filter_kwargs
         )
 
@@ -345,8 +364,7 @@ class ResultSetViewSet(viewsets.ViewSet):
             # of resultsets to be returned.
             del(rs_map[rs_id])
 
-            result_types = []
-            job_count = 0
+            job_counts = Counter()
 
             #itertools needs the elements to be sorted by the grouper
             by_platform = sorted(list(resultset_group), key=platform_grouper)
@@ -384,10 +402,10 @@ class ResultSetViewSet(viewsets.ViewSet):
                             kwargs={"project": jm.project, "pk": job["id"]})
 
                         if job["state"] == "completed":
-                            result_types.append(job["result"])
+                            job_counts[job["result"]] += 1
                         else:
-                            result_types.append(job["state"])
-                        job_count += 1
+                            job_counts[job["state"]] += 1
+                        job_counts["total"] += 1
 
                 platforms.append({
                     "name": platform_name,
@@ -399,8 +417,7 @@ class ResultSetViewSet(viewsets.ViewSet):
             #can be used to determine the resultset's severity
             resultset.update({
                 "platforms": platforms,
-                "result_types": list(set(result_types)),
-                "job_count": job_count,
+                "job_counts": job_counts,
             })
 
         # the resultsets left in the map have no jobs, so fill in the fields
@@ -408,8 +425,7 @@ class ResultSetViewSet(viewsets.ViewSet):
         for rs in rs_map.values():
             rs.update({
                 "platforms": [],
-                "result_types": [],
-                "job_count": 0,
+                "job_counts": {"total": 0},
             })
             resultsets.append(rs)
         return sorted(resultsets, key=lambda x: x["push_timestamp"], reverse=True)

From 04b1f2186f26af5036ff530dff440cbee007bd78 Mon Sep 17 00:00:00 2001
From: Cameron Dawson <cdawson@mozilla.com>
Date: Wed, 29 Jan 2014 14:35:54 -0800
Subject: [PATCH 03/11] sync resultset detail and list endpoints.  return
 push_timestamp with jobs

---
 treeherder/events/publisher.py |   4 +-
 treeherder/log_parser/tasks.py |   8 ++-
 treeherder/model/sql/jobs.json |  20 ++-----
 treeherder/webapp/api/views.py | 103 +++++++++++++++++++++++++--------
 4 files changed, 91 insertions(+), 44 deletions(-)

diff --git a/treeherder/events/publisher.py b/treeherder/events/publisher.py
index ab862fb61..71f4acb75 100644
--- a/treeherder/events/publisher.py
+++ b/treeherder/events/publisher.py
@@ -50,10 +50,10 @@ class EventsPublisher(object):
 
 class JobStatusPublisher(EventsPublisher):
 
-    def publish(self, job_id, result_set_id, branch, status):
+    def publish(self, job_id, resultset, branch, status):
         message = {
             "id": job_id,
-            "result_set_id": result_set_id,
+            "resultset": resultset,
             "event": "job",
             "branch": branch,
             "status": status
diff --git a/treeherder/log_parser/tasks.py b/treeherder/log_parser/tasks.py
index 7b65d43c7..4dcb2f0ef 100644
--- a/treeherder/log_parser/tasks.py
+++ b/treeherder/log_parser/tasks.py
@@ -34,6 +34,12 @@ def parse_log(project, job_id, result_set_id, check_errors=False):
     failure_publisher = JobFailurePublisher(settings.BROKER_URL)
 
     try:
+        # return the resultset with the job id to identify if the UI wants
+        # to fetch the whole thing.
+        resultset = jm.get_result_set_by_id(result_set_id=result_set_id)[0]
+        del(resultset["active_status"])
+        del(resultset["revision_hash"])
+
         log_references = jm.get_log_references(job_id)
 
         # we may have many log references per job
@@ -79,7 +85,7 @@ def parse_log(project, job_id, result_set_id, check_errors=False):
 
             # store the artifacts generated
             jm.store_job_artifact(artifact_list)
-        status_publisher.publish(job_id, result_set_id, project, 'processed')
+        status_publisher.publish(job_id, resultset, project, 'processed')
         if check_errors:
             failure_publisher.publish(job_id, project)
 
diff --git a/treeherder/model/sql/jobs.json b/treeherder/model/sql/jobs.json
index 768990e86..1a15271f2 100644
--- a/treeherder/model/sql/jobs.json
+++ b/treeherder/model/sql/jobs.json
@@ -227,8 +227,7 @@
 				  LEFT JOIN `REP0`.`job_group` as jg
                     ON jt.`job_group_id` = jg.id
                   WHERE 1
-                  REP1
-                  LIMIT ?,?",
+                  REP1",
 
             "host":"read_host"
         },
@@ -395,20 +394,9 @@
             "host": "read_host"
         },
         "get_result_set_by_id":{
-            "sql":"SELECT
-                      `rs`.`id`,
-                      `rs`.`revision_hash`,
-                      `rs`.`push_timestamp`,
-                      `rev`.`revision`,
-                      `rev`.`author`,
-                      `rev`.`repository_id`,
-                      `rev`.`comments`
-                   FROM result_set as rs
-                   LEFT JOIN revision_map as rm
-                       ON rs.id = rm.result_set_id
-                   LEFT JOIN revision as rev
-                       ON rm.revision_id = rev.id
-                   WHERE `rs`.`id` = ?",
+            "sql":"SELECT *
+                   FROM result_set
+                   WHERE id = ?",
             "host": "read_host"
         },
         "get_result_set_job_list":{
diff --git a/treeherder/webapp/api/views.py b/treeherder/webapp/api/views.py
index 60c071130..089ea9f05 100644
--- a/treeherder/webapp/api/views.py
+++ b/treeherder/webapp/api/views.py
@@ -193,7 +193,7 @@ class JobsViewSet(viewsets.ViewSet):
                 job["artifacts"].append(art)
 
             option_collections = jm.refdata_model.get_all_option_collections()
-            option_collections[job["option_collection_hash"]]['opt']
+            job["platform_opt"] = option_collections[job["option_collection_hash"]]['opt']
 
             return Response(job)
         else:
@@ -210,19 +210,9 @@ class JobsViewSet(viewsets.ViewSet):
 
         limit_condition = filters.pop("limit", set([("=", "0,10")])).pop()
         offset, limit = limit_condition[1].split(",")
-        objs = jm.get_job_list(offset, limit, filters)
-
-        offset = int(request.QUERY_PARAMS.get('offset', 0))
-        count = int(request.QUERY_PARAMS.get('count', 10))
         full = request.QUERY_PARAMS.get('full', 'true').lower() == 'true'
+        objs = jm.get_job_list(offset, limit, full, filters)
 
-        objs = jm.get_job_list(
-            offset,
-            count,
-            full,
-            **dict((k, v) for k, v in request.QUERY_PARAMS.iteritems()
-                   if k in filters)
-        )
         if objs:
             option_collections = jm.refdata_model.get_all_option_collections()
             for job in objs:
@@ -273,6 +263,55 @@ class JobsViewSet(viewsets.ViewSet):
         return Response({'message': 'Job successfully updated'})
 
 
+class RevisionViewSet(viewsets.ViewSet):
+    """
+    View for ``revision`` records
+
+    ``result sets`` are synonymous with ``pushes`` in the ui
+    """
+
+    @with_jobs
+    def list(self, request, project, jm):
+        """
+        GET method for list of ``resultset`` records with revisions
+
+        """
+
+        filters = UrlQueryFilter(request.QUERY_PARAMS).parse()
+
+        limit_condition = filters.pop("limit", set([("=", "0,10")])).pop()
+        offset, limit = limit_condition[1].split(",")
+        full = request.QUERY_PARAMS.get('full', "true").lower() == "true"
+
+        objs = jm.get_result_set_list(
+            offset,
+            limit,
+            full,
+            filters
+        )
+        return Response(objs)
+
+    @with_jobs
+    def retrieve(self, request, project, jm, pk=None):
+        """
+        GET method implementation for detail view of ``resultset``
+        """
+        filters = ["job_type_name"]
+
+        full = request.QUERY_PARAMS.get('full', "true").lower() == "true"
+
+        filter_kwargs = dict(
+            (k, v) for k, v in request.QUERY_PARAMS.iteritems()
+            if k in filters
+        )
+
+        rev = jm.get_revision_by_id(pk)
+        if rev:
+            return Response(rev[0])
+        else:
+            return Response("No resultset with id: {0}".format(pk), 404)
+
+
 class ResultSetViewSet(viewsets.ViewSet):
     """
     View for ``resultset`` records
@@ -306,22 +345,32 @@ class ResultSetViewSet(viewsets.ViewSet):
         """
         GET method implementation for detail view of ``resultset``
         """
-        filters = ["job_type_name"]
+        filters = UrlQueryFilter({"id": pk}).parse()
 
         full = request.QUERY_PARAMS.get('full', "true").lower() == "true"
 
-        filter_kwargs = dict(
-            (k, v) for k, v in request.QUERY_PARAMS.iteritems()
-            if k in filters
-        )
-
-        rs = jm.get_result_set_by_id(pk)
-        if rs:
-            resultsets = self.get_resultsets_with_jobs(jm, [rs[0]], full, filter_kwargs)
-            return Response(resultsets[0])
+        objs = jm.get_result_set_list(0, 1, full, filters)
+        if objs:
+            rs = self.get_resultsets_with_jobs(jm, objs, full, {})
+            return Response(rs[0])
         else:
             return Response("No resultset with id: {0}".format(pk), 404)
 
+    @staticmethod
+    def get_job_counter():
+        return Counter({
+            "busted": 0,
+            "exception": 0,
+            "testfailed": 0,
+            "unknown": 0,
+            "usercancel": 0,
+            "retry": 0,
+            "success": 0,
+            "running": 0,
+            "pending": 0,
+            "total": 0
+        })
+
     @staticmethod
     def get_resultsets_with_jobs(jm, rs_list, full, filter_kwargs):
         """Convert db result of resultsets in a list to JSON"""
@@ -364,7 +413,7 @@ class ResultSetViewSet(viewsets.ViewSet):
             # of resultsets to be returned.
             del(rs_map[rs_id])
 
-            job_counts = Counter()
+            job_counts = ResultSetViewSet.get_job_counter()
 
             #itertools needs the elements to be sorted by the grouper
             by_platform = sorted(list(resultset_group), key=platform_grouper)
@@ -397,6 +446,7 @@ class ResultSetViewSet(viewsets.ViewSet):
                         job["id"] = job["job_id"]
                         del(job["job_id"])
                         del(job["result_set_id"])
+                        del(job["option_collection_hash"])
 
                         job["resource_uri"] = reverse("jobs-detail",
                             kwargs={"project": jm.project, "pk": job["id"]})
@@ -425,10 +475,13 @@ class ResultSetViewSet(viewsets.ViewSet):
         for rs in rs_map.values():
             rs.update({
                 "platforms": [],
-                "job_counts": {"total": 0},
+                "job_counts": ResultSetViewSet.get_job_counter(),
             })
             resultsets.append(rs)
-        return sorted(resultsets, key=lambda x: x["push_timestamp"], reverse=True)
+        return sorted(
+            resultsets,
+            key=lambda x: x["push_timestamp"],
+            reverse=True)
 
 
     @with_jobs

From 8f0e344f2bfdbb1ded8419137b061523da235e61 Mon Sep 17 00:00:00 2001
From: Cameron Dawson <cdawson@mozilla.com>
Date: Wed, 29 Jan 2014 16:27:11 -0800
Subject: [PATCH 04/11] create endpoint for resultset revisions

---
 treeherder/model/derived/jobs.py | 14 ++++++++++++++
 treeherder/webapp/api/views.py   | 12 ++++++++++++
 2 files changed, 26 insertions(+)

diff --git a/treeherder/model/derived/jobs.py b/treeherder/model/derived/jobs.py
index 356d6fbe4..10e1e1d19 100644
--- a/treeherder/model/derived/jobs.py
+++ b/treeherder/model/derived/jobs.py
@@ -346,6 +346,20 @@ class JobsModel(TreeherderModelBase):
         return lookups
 
 
+    def get_resultset_revisions_list(self, result_set_id):
+        """
+        Return the revisions for the given resultset
+        """
+
+        proc = "jobs.selects.get_result_set_details"
+        lookups = self.get_jobs_dhub().execute(
+            proc=proc,
+            debug_show=self.DEBUG,
+            replace=[result_set_id],
+        )
+        return lookups
+
+
     def get_result_set_details(self, result_set_ids):
         """
         Retrieve all revisions associated with a set of ``result_set``
diff --git a/treeherder/webapp/api/views.py b/treeherder/webapp/api/views.py
index 089ea9f05..8359fa284 100644
--- a/treeherder/webapp/api/views.py
+++ b/treeherder/webapp/api/views.py
@@ -356,6 +356,15 @@ class ResultSetViewSet(viewsets.ViewSet):
         else:
             return Response("No resultset with id: {0}".format(pk), 404)
 
+    @link()
+    @with_jobs
+    def revisions(self, request, project, jm, pk=None):
+        """
+        GET method for revisions of a resultset
+        """
+        objs = jm.get_resultset_revisions_list(pk)
+        return Response(objs)
+
     @staticmethod
     def get_job_counter():
         return Counter({
@@ -404,6 +413,9 @@ class ResultSetViewSet(viewsets.ViewSet):
         for rs_id, resultset_group in itertools.groupby(rs_sorted, key=rs_grouper):
 
             resultset = rs_map[rs_id]
+            resultset["revisions_uri"] = reverse("resultset-revisions",
+                    kwargs={"project": jm.project, "pk": rs_id})
+
             resultsets.append(resultset)
 
             # we found jobs for this resultset, so remove it from the map

From a79255cb72bb7c24122a7722d8aab804238dfab6 Mon Sep 17 00:00:00 2001
From: Cameron Dawson <cdawson@mozilla.com>
Date: Thu, 30 Jan 2014 09:40:29 -0800
Subject: [PATCH 05/11] ensure ``revisions_uri`` is on all resultsets, even if
 no jobs

---
 treeherder/webapp/api/views.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/treeherder/webapp/api/views.py b/treeherder/webapp/api/views.py
index 8359fa284..ffd58e765 100644
--- a/treeherder/webapp/api/views.py
+++ b/treeherder/webapp/api/views.py
@@ -389,6 +389,9 @@ class ResultSetViewSet(viewsets.ViewSet):
         rs_map = {}
         for rs in rs_list:
             rs_map[rs["id"]] = rs
+            # all rs should have the revisions_uri, so add it here
+            rs["revisions_uri"] = reverse("resultset-revisions",
+                kwargs={"project": jm.project, "pk": rs["id"]})
 
         jobs_ungrouped = jm.get_result_set_job_list(
             rs_map.keys(),
@@ -413,9 +416,6 @@ class ResultSetViewSet(viewsets.ViewSet):
         for rs_id, resultset_group in itertools.groupby(rs_sorted, key=rs_grouper):
 
             resultset = rs_map[rs_id]
-            resultset["revisions_uri"] = reverse("resultset-revisions",
-                    kwargs={"project": jm.project, "pk": rs_id})
-
             resultsets.append(resultset)
 
             # we found jobs for this resultset, so remove it from the map
@@ -502,7 +502,7 @@ class ResultSetViewSet(viewsets.ViewSet):
         POST method implementation
         """
         try:
-            jm.store_result_set_data( request.DATA )
+            jm.store_result_set_data(request.DATA)
         except DatasetNotFoundError as e:
             return Response({"message": str(e)}, status=404)
         except Exception as e:  # pragma nocover
@@ -515,7 +515,6 @@ class ResultSetViewSet(viewsets.ViewSet):
         return Response({"message": "well-formed JSON stored"})
 
 
-
 class RevisionLookupSetViewSet(viewsets.ViewSet):
 
     @with_jobs

From 28d39b6b83c7a919c287ac95c001e44b3d8c0b22 Mon Sep 17 00:00:00 2001
From: Cameron Dawson <cdawson@mozilla.com>
Date: Thu, 30 Jan 2014 11:17:55 -0800
Subject: [PATCH 06/11] don't need Counter class

---
 treeherder/webapp/api/views.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/treeherder/webapp/api/views.py b/treeherder/webapp/api/views.py
index ffd58e765..48f83162e 100644
--- a/treeherder/webapp/api/views.py
+++ b/treeherder/webapp/api/views.py
@@ -1,8 +1,6 @@
 import simplejson as json
 import itertools
 
-from collections import Counter
-
 from django.conf import settings
 from rest_framework import viewsets
 from rest_framework.response import Response
@@ -367,7 +365,7 @@ class ResultSetViewSet(viewsets.ViewSet):
 
     @staticmethod
     def get_job_counter():
-        return Counter({
+        return {
             "busted": 0,
             "exception": 0,
             "testfailed": 0,
@@ -378,7 +376,7 @@ class ResultSetViewSet(viewsets.ViewSet):
             "running": 0,
             "pending": 0,
             "total": 0
-        })
+        }
 
     @staticmethod
     def get_resultsets_with_jobs(jm, rs_list, full, filter_kwargs):

From 00df9bf6fca9d67344eb775b536ae23b9bb454fb Mon Sep 17 00:00:00 2001
From: Cameron Dawson <cdawson@mozilla.com>
Date: Thu, 30 Jan 2014 13:24:02 -0800
Subject: [PATCH 07/11] test fixes

---
 tests/model/derived/test_objectstore_model.py | 4 ++--
 tests/webapp/api/test_jobs_api.py             | 8 ++++----
 tests/webapp/api/test_resultset_api.py        | 7 ++++---
 treeherder/model/derived/jobs.py              | 1 -
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/tests/model/derived/test_objectstore_model.py b/tests/model/derived/test_objectstore_model.py
index e8dc70be9..64225b996 100644
--- a/tests/model/derived/test_objectstore_model.py
+++ b/tests/model/derived/test_objectstore_model.py
@@ -150,7 +150,7 @@ def test_objectstore_update_content(jm, sample_data):
     jm.store_job_data([original_obj])
 
     obj_updated = original_obj.copy()
-    obj_updated["job"]["state"] = "new_state"
+    obj_updated["job"]["state"] = "derp_state"
 
     jm.store_job_data([obj_updated])
 
@@ -165,4 +165,4 @@ def test_objectstore_update_content(jm, sample_data):
     stored_blob = json.loads(stored_objs[0]["json_blob"])
 
     # check that the blob was updated
-    assert stored_blob["job"]["state"] == "new_state"
+    assert stored_blob["job"]["state"] == "derp_state"
diff --git a/tests/webapp/api/test_jobs_api.py b/tests/webapp/api/test_jobs_api.py
index d08164adf..57db56735 100644
--- a/tests/webapp/api/test_jobs_api.py
+++ b/tests/webapp/api/test_jobs_api.py
@@ -207,10 +207,10 @@ def test_job_detail_not_found(webapp, jm):
 
 
 def test_retrieve_result_set(jm, webapp, eleven_jobs_processed):
-    resp = webapp.get(
-        reverse("resultset-list",
-                kwargs={"project": jm.project})
-    )
+    url = reverse("resultset-list", kwargs={"project": jm.project})
+    print url
+    resp = webapp.get(url)
+
     assert resp.status_int == 200
     assert isinstance(resp.json, list)
 
diff --git a/tests/webapp/api/test_resultset_api.py b/tests/webapp/api/test_resultset_api.py
index 6d027bc2b..e2edc9a08 100644
--- a/tests/webapp/api/test_resultset_api.py
+++ b/tests/webapp/api/test_resultset_api.py
@@ -26,9 +26,10 @@ def test_resultset_list(webapp, eleven_jobs_processed, jm):
         u'revision_hash',
         u'revision',
         u'revision_list',
-        u'job_count',
-        u'platforms',
-        u'result_types'
+        u'revision_count',
+        u'revisions_uri',
+        u'job_counts',
+        u'platforms'
     ])
     for rs in rs_list:
         assert set(rs.keys()) == exp_keys
diff --git a/treeherder/model/derived/jobs.py b/treeherder/model/derived/jobs.py
index 10e1e1d19..40525ca57 100644
--- a/treeherder/model/derived/jobs.py
+++ b/treeherder/model/derived/jobs.py
@@ -318,7 +318,6 @@ class JobsModel(TreeherderModelBase):
             }
             if full:
                 list_item.update({
-                    "full": full,
                     "comments": detail['comments'],
                     "revision_list": aggregate_details[result['id']]
                 })

From c6ec4cf7c53875c57849fbbac2dcb896d21cf5aa Mon Sep 17 00:00:00 2001
From: Cameron Dawson <cdawson@mozilla.com>
Date: Thu, 30 Jan 2014 13:33:34 -0800
Subject: [PATCH 08/11] fix for unexpected job state

---
 tests/model/derived/test_objectstore_model.py | 4 ++--
 treeherder/webapp/api/views.py                | 8 ++++++--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/tests/model/derived/test_objectstore_model.py b/tests/model/derived/test_objectstore_model.py
index 64225b996..e8dc70be9 100644
--- a/tests/model/derived/test_objectstore_model.py
+++ b/tests/model/derived/test_objectstore_model.py
@@ -150,7 +150,7 @@ def test_objectstore_update_content(jm, sample_data):
     jm.store_job_data([original_obj])
 
     obj_updated = original_obj.copy()
-    obj_updated["job"]["state"] = "derp_state"
+    obj_updated["job"]["state"] = "new_state"
 
     jm.store_job_data([obj_updated])
 
@@ -165,4 +165,4 @@ def test_objectstore_update_content(jm, sample_data):
     stored_blob = json.loads(stored_objs[0]["json_blob"])
 
     # check that the blob was updated
-    assert stored_blob["job"]["state"] == "derp_state"
+    assert stored_blob["job"]["state"] == "new_state"
diff --git a/treeherder/webapp/api/views.py b/treeherder/webapp/api/views.py
index 48f83162e..1b679f352 100644
--- a/treeherder/webapp/api/views.py
+++ b/treeherder/webapp/api/views.py
@@ -1,6 +1,8 @@
 import simplejson as json
 import itertools
 
+from collections import defaultdict
+
 from django.conf import settings
 from rest_framework import viewsets
 from rest_framework.response import Response
@@ -365,7 +367,8 @@ class ResultSetViewSet(viewsets.ViewSet):
 
     @staticmethod
     def get_job_counter():
-        return {
+        d = defaultdict(int)
+        d.update({
             "busted": 0,
             "exception": 0,
             "testfailed": 0,
@@ -376,7 +379,8 @@ class ResultSetViewSet(viewsets.ViewSet):
             "running": 0,
             "pending": 0,
             "total": 0
-        }
+        })
+        return d
 
     @staticmethod
     def get_resultsets_with_jobs(jm, rs_list, full, filter_kwargs):

From 8ecde732a3d9fad4c24a213136b1dbb24263f2ed Mon Sep 17 00:00:00 2001
From: Cameron Dawson <cdawson@mozilla.com>
Date: Thu, 30 Jan 2014 15:43:51 -0800
Subject: [PATCH 09/11] code cleanup

---
 tests/webapp/api/test_jobs_api.py |  8 ++---
 treeherder/webapp/api/views.py    | 49 -------------------------------
 2 files changed, 4 insertions(+), 53 deletions(-)

diff --git a/tests/webapp/api/test_jobs_api.py b/tests/webapp/api/test_jobs_api.py
index 57db56735..d08164adf 100644
--- a/tests/webapp/api/test_jobs_api.py
+++ b/tests/webapp/api/test_jobs_api.py
@@ -207,10 +207,10 @@ def test_job_detail_not_found(webapp, jm):
 
 
 def test_retrieve_result_set(jm, webapp, eleven_jobs_processed):
-    url = reverse("resultset-list", kwargs={"project": jm.project})
-    print url
-    resp = webapp.get(url)
-
+    resp = webapp.get(
+        reverse("resultset-list",
+                kwargs={"project": jm.project})
+    )
     assert resp.status_int == 200
     assert isinstance(resp.json, list)
 
diff --git a/treeherder/webapp/api/views.py b/treeherder/webapp/api/views.py
index 1b679f352..781b9e67c 100644
--- a/treeherder/webapp/api/views.py
+++ b/treeherder/webapp/api/views.py
@@ -263,55 +263,6 @@ class JobsViewSet(viewsets.ViewSet):
         return Response({'message': 'Job successfully updated'})
 
 
-class RevisionViewSet(viewsets.ViewSet):
-    """
-    View for ``revision`` records
-
-    ``result sets`` are synonymous with ``pushes`` in the ui
-    """
-
-    @with_jobs
-    def list(self, request, project, jm):
-        """
-        GET method for list of ``resultset`` records with revisions
-
-        """
-
-        filters = UrlQueryFilter(request.QUERY_PARAMS).parse()
-
-        limit_condition = filters.pop("limit", set([("=", "0,10")])).pop()
-        offset, limit = limit_condition[1].split(",")
-        full = request.QUERY_PARAMS.get('full', "true").lower() == "true"
-
-        objs = jm.get_result_set_list(
-            offset,
-            limit,
-            full,
-            filters
-        )
-        return Response(objs)
-
-    @with_jobs
-    def retrieve(self, request, project, jm, pk=None):
-        """
-        GET method implementation for detail view of ``resultset``
-        """
-        filters = ["job_type_name"]
-
-        full = request.QUERY_PARAMS.get('full', "true").lower() == "true"
-
-        filter_kwargs = dict(
-            (k, v) for k, v in request.QUERY_PARAMS.iteritems()
-            if k in filters
-        )
-
-        rev = jm.get_revision_by_id(pk)
-        if rev:
-            return Response(rev[0])
-        else:
-            return Response("No resultset with id: {0}".format(pk), 404)
-
-
 class ResultSetViewSet(viewsets.ViewSet):
     """
     View for ``resultset`` records

From d787175340838d715fad2f7ae04076a9f53ff399 Mon Sep 17 00:00:00 2001
From: Cameron Dawson <cdawson@mozilla.com>
Date: Fri, 31 Jan 2014 10:37:48 -0800
Subject: [PATCH 10/11] update for results and statuses in jobsmodel.  remove
 update_state endpoint

---
 tests/webapp/api/test_jobs_api.py | 77 -------------------------------
 treeherder/etl/buildbot.py        |  1 -
 treeherder/model/derived/jobs.py  | 13 +++++-
 treeherder/webapp/api/views.py    | 53 ++-------------------
 4 files changed, 16 insertions(+), 128 deletions(-)

diff --git a/tests/webapp/api/test_jobs_api.py b/tests/webapp/api/test_jobs_api.py
index d08164adf..a224d2fa8 100644
--- a/tests/webapp/api/test_jobs_api.py
+++ b/tests/webapp/api/test_jobs_api.py
@@ -3,83 +3,6 @@ import json
 from django.core.urlresolvers import reverse
 
 
-def test_update_state_success(webapp, eleven_jobs_processed, jm):
-    """
-    test setting the state of a job via webtest.
-    extected result are:
-    - return code 200
-    - return message successful
-    - job status updated
-    """
-
-    job = jm.get_job_list(0, 1)[0]
-    job_id = job["id"]
-    new_state = "coalesced"
-
-    # use the backdoor to set the state of the job to something we can
-    # change.  because we can't change it once it's ``completed``
-    jm.get_jobs_dhub().execute(
-        proc='jobs_test.updates.set_state_any',
-        placeholders=["running", job_id],
-    )
-
-    url = reverse("jobs-update-state", kwargs={
-        "project": jm.project,
-        "pk": job_id
-    })
-
-    resp = webapp.post(url, params={"state": new_state})
-
-    assert resp.status_int == 200
-    assert resp.json["message"] == "state updated to '{0}'".format(new_state)
-    assert jm.get_job(job_id)[0]["state"] == new_state
-
-
-def test_update_state_invalid_state(webapp, eleven_jobs_processed, jm):
-    """
-    test setting the state of a job via webtest with invalid state.
-    extected result are:
-    - return code 400
-    """
-
-    job = jm.get_job_list(0, 1)[0]
-    job_id = job["id"]
-    old_state = job["state"]
-    new_state = "chokey"
-
-    url = reverse("jobs-update-state", kwargs={
-        "project": jm.project,
-        "pk": job_id
-    })
-
-    resp = webapp.post(url, params={"state": new_state}, status=400)
-
-    assert resp.json["message"] == ("'{0}' is not a valid state.  Must be "
-                                    "one of: {1}".format(
-                                        new_state,
-                                        ", ".join(jm.STATES)
-                                    ))
-    assert jm.get_job(job_id)[0]["state"] == old_state
-
-
-def test_update_state_invalid_job_id(webapp, eleven_jobs_processed, jm):
-    """
-    test setting the state of a job via webtest with invalid job_id.
-    extected result are:
-    - return code 404
-    """
-
-    job_id = -32767
-    new_state = "pending"
-
-    url = reverse("jobs-update-state", kwargs={
-        "project": jm.project,
-        "pk": job_id
-    })
-
-    webapp.post(url, params={"state": new_state}, status=404)
-
-
 def test_job_list(webapp, eleven_jobs_processed, jm):
     """
     test retrieving a list of ten json blobs from the jobs-list
diff --git a/treeherder/etl/buildbot.py b/treeherder/etl/buildbot.py
index feac750df..083fde53f 100644
--- a/treeherder/etl/buildbot.py
+++ b/treeherder/etl/buildbot.py
@@ -10,7 +10,6 @@ RESULT_DICT = {
     6: "usercancel"
 }
 
-
 ####
 #   The following variables were taken from util.py
 #
diff --git a/treeherder/model/derived/jobs.py b/treeherder/model/derived/jobs.py
index 40525ca57..e0b2b3cb1 100644
--- a/treeherder/model/derived/jobs.py
+++ b/treeherder/model/derived/jobs.py
@@ -26,7 +26,18 @@ class JobsModel(TreeherderModelBase):
     CT_JOBS = "jobs"
     CT_OBJECTSTORE = "objectstore"
     CONTENT_TYPES = [CT_JOBS, CT_OBJECTSTORE]
-    STATES = ["pending", "running", "completed", "coalesced"]
+    RESULTS = [
+        "busted",
+        "exception",
+        "testfailed",
+        "unknown",
+        "usercancel",
+        "retry",
+        "success",
+    ]
+    INCOMPLETE_STATES = ["running", "pending"]
+    STATES = INCOMPLETE_STATES + ["completed", "coalesced"]
+
     # list of searchable columns, i.e. those who have an index
     # it would be nice to get this directly from the db and cache it
     INDEXED_COLUMNS = {
diff --git a/treeherder/webapp/api/views.py b/treeherder/webapp/api/views.py
index 17277c957..dde708d66 100644
--- a/treeherder/webapp/api/views.py
+++ b/treeherder/webapp/api/views.py
@@ -199,7 +199,6 @@ class JobsViewSet(viewsets.ViewSet):
         else:
             return Response("No job with id: {0}".format(pk), 404)
 
-
     @with_jobs
     def list(self, request, project, jm):
         """
@@ -221,35 +220,6 @@ class JobsViewSet(viewsets.ViewSet):
 
         return Response(objs)
 
-    @action()
-    @with_jobs
-    def update_state(self, request, project, jm, pk=None):
-        """
-        Change the state of a job.
-        """
-        state = request.DATA.get('state', None)
-
-        # check that this state is valid
-        if state not in jm.STATES:
-            return Response(
-                {"message": ("'{0}' is not a valid state.  Must be "
-                             "one of: {1}".format(
-                                 state,
-                                 ", ".join(jm.STATES)
-                             ))},
-                status=400,
-            )
-
-        if not pk:  # pragma nocover
-            return Response({"message": "job id required"}, status=400)
-
-        obj = jm.get_job(pk)
-        if obj:
-            jm.set_state(pk, state)
-            return Response({"message": "state updated to '{0}'".format(state)})
-        else:
-            return Response("No job with id: {0}".format(pk), 404)
-
     @with_jobs
     def create(self, request, project, jm):
         """
@@ -315,23 +285,6 @@ class ResultSetViewSet(viewsets.ViewSet):
         objs = jm.get_resultset_revisions_list(pk)
         return Response(objs)
 
-    @staticmethod
-    def get_job_counter():
-        d = defaultdict(int)
-        d.update({
-            "busted": 0,
-            "exception": 0,
-            "testfailed": 0,
-            "unknown": 0,
-            "usercancel": 0,
-            "retry": 0,
-            "success": 0,
-            "running": 0,
-            "pending": 0,
-            "total": 0
-        })
-        return d
-
     @staticmethod
     def get_resultsets_with_jobs(jm, rs_list, full, filter_kwargs):
         """Convert db result of resultsets in a list to JSON"""
@@ -377,7 +330,8 @@ class ResultSetViewSet(viewsets.ViewSet):
             # of resultsets to be returned.
             del(rs_map[rs_id])
 
-            job_counts = ResultSetViewSet.get_job_counter()
+            job_counts = dict.fromkeys(
+                jm.RESULTS + jm.INCOMPLETE_STATES + ["total"], 0)
 
             #itertools needs the elements to be sorted by the grouper
             by_platform = sorted(list(resultset_group), key=platform_grouper)
@@ -439,7 +393,8 @@ class ResultSetViewSet(viewsets.ViewSet):
         for rs in rs_map.values():
             rs.update({
                 "platforms": [],
-                "job_counts": ResultSetViewSet.get_job_counter(),
+                "job_counts": dict.fromkeys(
+                    jm.RESULTS + jm.INCOMPLETE_STATES + ["total"], 0),
             })
             resultsets.append(rs)
         return sorted(

From f8f4aa3e321d6ebad1737d719c447707c77adade Mon Sep 17 00:00:00 2001
From: Cameron Dawson <cdawson@mozilla.com>
Date: Fri, 31 Jan 2014 11:36:33 -0800
Subject: [PATCH 11/11] fix tests

---
 tests/model/derived/test_objectstore_model.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/model/derived/test_objectstore_model.py b/tests/model/derived/test_objectstore_model.py
index e8dc70be9..7eda8fb9b 100644
--- a/tests/model/derived/test_objectstore_model.py
+++ b/tests/model/derived/test_objectstore_model.py
@@ -146,11 +146,11 @@ def test_objectstore_update_content(jm, sample_data):
     """
     Test updating an object of the objectstore.
     """
-    original_obj = sample_data.job_data[0]
+    original_obj = sample_data.job_data[100]
     jm.store_job_data([original_obj])
 
     obj_updated = original_obj.copy()
-    obj_updated["job"]["state"] = "new_state"
+    obj_updated["job"]["state"] = "pending"
 
     jm.store_job_data([obj_updated])
 
@@ -165,4 +165,4 @@ def test_objectstore_update_content(jm, sample_data):
     stored_blob = json.loads(stored_objs[0]["json_blob"])
 
     # check that the blob was updated
-    assert stored_blob["job"]["state"] == "new_state"
+    assert stored_blob["job"]["state"] == "pending"