From 1b7fc1d546513b50fc59936fc7400b778abd1576 Mon Sep 17 00:00:00 2001
From: Cameron Dawson <cdawson@mozilla.com>
Date: Tue, 1 Apr 2014 09:44:27 -0700
Subject: [PATCH 1/7] begin support of rs data ranges

---
 treeherder/model/derived/jobs.py   | 25 ++++++++++++++++++++++++-
 treeherder/webapp/api/resultset.py | 14 +++++++++++++-
 treeherder/webapp/api/utils.py     |  6 +++++-
 3 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/treeherder/model/derived/jobs.py b/treeherder/model/derived/jobs.py
index 9501b26e4..9d5e46b81 100644
--- a/treeherder/model/derived/jobs.py
+++ b/treeherder/model/derived/jobs.py
@@ -65,7 +65,9 @@ class JobsModel(TreeherderModelBase):
             "id": "rs.id",
             "revision_hash": "rs.revision_hash",
             "revision": "revision.revision",
-            "author": "revision.author"
+            "author": "revision.author",
+            "push_timestamp": "rs.push_timestamp",
+            "revision_date": "revision.commit_timestamp"
         },
         "bug_job_map": {
             "job_id": "job_id",
@@ -587,6 +589,27 @@ class JobsModel(TreeherderModelBase):
         )
         return data
 
+    def get_push_timestamp_lookup_for_revisions(self, revisions):
+        """Get the push timestamp for a list of revisions."""
+
+        # Generate a list of result_set_ids
+        id_placeholders = []
+        repl = []
+        for data in result_set_ids:
+            id_placeholders.append('%s')
+        repl.append(','.join(id_placeholders))
+
+        proc = "jobs.selects.get_result_set_push_timestamp"
+        data = self.get_jobs_dhub().execute(
+            proc=proc,
+            placeholders=result_set_ids,
+            debug_show=self.DEBUG,
+            replace=repl,
+            return_type="dict",
+            key_column="id"
+        )
+        return data
+
     ##################
     #
     # Objectstore functionality
diff --git a/treeherder/webapp/api/resultset.py b/treeherder/webapp/api/resultset.py
index 17b4fd544..891287a14 100644
--- a/treeherder/webapp/api/resultset.py
+++ b/treeherder/webapp/api/resultset.py
@@ -1,4 +1,6 @@
 import itertools
+import time
+import datetime
 
 from rest_framework import viewsets
 from rest_framework.response import Response
@@ -22,8 +24,18 @@ class ResultSetViewSet(viewsets.ViewSet):
         GET method for list of ``resultset`` records with revisions
 
         """
+        def xlate_date(datestr):
+            # exp a date like 2014-03-31.  change to timestamp
+            return time.mktime(datetime.datetime.strptime(datestr, "%Y-%m-%d").timetuple())
 
-        filter = UrlQueryFilter(request.QUERY_PARAMS)
+        filter = UrlQueryFilter(request.QUERY_PARAMS, {
+            "push_timestamp": xlate_date,
+        })
+
+        # translate range filter conditions for date and revision
+
+        # fromchange = filter.pop("fromchange", None)
+        # tochange = filter.pop("tochange", None)
 
         offset_id = filter.pop("id__lt", 0)
         count = filter.pop("count", 10)
diff --git a/treeherder/webapp/api/utils.py b/treeherder/webapp/api/utils.py
index e5b738ce4..542dfa4b3 100644
--- a/treeherder/webapp/api/utils.py
+++ b/treeherder/webapp/api/utils.py
@@ -51,7 +51,7 @@ class UrlQueryFilter(object):
 
     splitter = "__"
 
-    def __init__(self, query_params):
+    def __init__(self, query_params, translators):
         self.raw_params = query_params
         self.conditions = defaultdict(set)
         for k, v in self.raw_params.iteritems():
@@ -65,6 +65,10 @@ class UrlQueryFilter(object):
                 field = k
                 operator = "="
 
+            # run the value through the translator for this field
+            if field in translators.keys():
+                v = translators[field](v)
+
             self.conditions[field].add((self.operators[operator], v))
 
     def get(self, key, default=None):

From d9429382cb868b35dc11b16350a2a773b07ab3cc Mon Sep 17 00:00:00 2001
From: Cameron Dawson <cdawson@mozilla.com>
Date: Tue, 1 Apr 2014 12:06:04 -0700
Subject: [PATCH 2/7] adding supervisord logs to ignore list

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index fb9721855..0c63262d8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -28,6 +28,7 @@ test.log
 treeherder*.log
 treeherder.log.*
 LOGFILE
+supervisor*.log
 
 # Unit test / coverage reports
 .coverage

From 0bb6b73b672dd5d6dbe3f2bffb03a8c31ab4e7bc Mon Sep 17 00:00:00 2001
From: Cameron Dawson <cdawson@mozilla.com>
Date: Wed, 2 Apr 2014 11:03:32 -0700
Subject: [PATCH 3/7] support for date and revision ranges

---
 tests/webapp/api/test_bug_job_map_api.py | 36 ++++++++++++++++++--
 tests/webapp/api/test_resultset_api.py   | 42 ++++++++++++++++++++++-
 treeherder/model/derived/jobs.py         | 24 +------------
 treeherder/webapp/api/bug.py             | 15 +++++++--
 treeherder/webapp/api/resultset.py       | 43 +++++++++++++++++-------
 treeherder/webapp/api/utils.py           | 19 +++++++++--
 6 files changed, 136 insertions(+), 43 deletions(-)

diff --git a/tests/webapp/api/test_bug_job_map_api.py b/tests/webapp/api/test_bug_job_map_api.py
index 06787b9a5..ab870904e 100644
--- a/tests/webapp/api/test_bug_job_map_api.py
+++ b/tests/webapp/api/test_bug_job_map_api.py
@@ -47,7 +47,39 @@ def test_create_bug_job_map(eleven_jobs_processed, mock_message_broker, jm):
         reverse("bug-job-map-list", kwargs={"project": jm.project}),
         bug_job_map_obj
     )
-    
+
+    user.delete()
+
+    assert (bug_job_map_obj,) == jm.get_bug_job_map_list(0, 1)
+
+
+def test_create_bug_job_map_dup(eleven_jobs_processed, mock_message_broker, jm):
+    """
+    test creating the same bug map skips it
+    """
+
+    client = APIClient()
+    user = User.objects.create(username="MyName", is_staff=True)
+    client.force_authenticate(user=user)
+
+    job = jm.get_job_list(0, 1)[0]
+
+    bug_job_map_obj = {
+        "job_id": job["id"],
+        "bug_id": 1,
+        "type": "manual"
+    }
+
+    client.post(
+        reverse("bug-job-map-list", kwargs={"project": jm.project}),
+        bug_job_map_obj
+    )
+
+    client.post(
+        reverse("bug-job-map-list", kwargs={"project": jm.project}),
+        bug_job_map_obj
+    )
+
     user.delete()
 
     assert (bug_job_map_obj,) == jm.get_bug_job_map_list(0, 1)
@@ -122,7 +154,7 @@ def test_bug_job_map_delete(webapp, eleven_jobs_processed,
             "pk": pk
         })
     )
-    
+
     user.delete()
 
     content = json.loads(resp.content)
diff --git a/tests/webapp/api/test_resultset_api.py b/tests/webapp/api/test_resultset_api.py
index 1608d1bc6..29706550d 100644
--- a/tests/webapp/api/test_resultset_api.py
+++ b/tests/webapp/api/test_resultset_api.py
@@ -1,10 +1,10 @@
 import pytest
 from django.core.urlresolvers import reverse
-from treeherder.webapp.api.resultset import ResultSetViewSet
 
 from thclient import TreeherderResultSetCollection
 from tests import test_utils
 
+from treeherder.webapp.api import utils
 
 def test_resultset_list(webapp, eleven_jobs_processed, jm):
     """
@@ -67,6 +67,46 @@ def test_resultset_list_empty_rs_still_show(webapp, initial_data,
     assert len(resp.json) == 10
 
 
+def test_resultset_list_filter_by_revision(webapp, eleven_jobs_processed, jm):
+    """
+    test retrieving a resultset list, filtered by a date range
+    """
+
+    resp = webapp.get(
+        reverse("resultset-list", kwargs={"project": jm.project}),
+        {"fromchange": "21fb3eed1b5f", "tochange": "909f55c626a8"}
+    )
+    assert resp.status_int == 200
+    assert len(resp.json) == 4
+    assert set([rs["revision"] for rs in resp.json]) == set(
+        ["909f55c626a8","71d49fee325a","bb57e9f67223","21fb3eed1b5f"]
+        )
+
+
+def test_resultset_list_filter_by_date(webapp, initial_data,
+                                       sample_resultset, jm):
+    """
+    test retrieving a resultset list, filtered by a date range
+    """
+    sample_resultset[3]["push_timestamp"] = utils.to_timestamp("2013-08-09")
+    sample_resultset[4]["push_timestamp"] = utils.to_timestamp("2013-08-10")
+    sample_resultset[5]["push_timestamp"] = utils.to_timestamp("2013-08-11")
+    sample_resultset[6]["push_timestamp"] = utils.to_timestamp("2013-08-12")
+    sample_resultset[7]["push_timestamp"] = utils.to_timestamp("2013-08-13")
+
+    jm.store_result_set_data(sample_resultset)
+
+    resp = webapp.get(
+        reverse("resultset-list", kwargs={"project": jm.project}),
+        {"startdate": "2013-08-10", "enddate": "2013-08-13"}
+    )
+    assert resp.status_int == 200
+    assert len(resp.json) == 4
+    assert set([rs["revision"] for rs in resp.json]) == set(
+        ["909f55c626a8","71d49fee325a","bb57e9f67223","668424578a0d"]
+        )
+
+
 def test_resultset_detail(webapp, eleven_jobs_processed, jm):
     """
     test retrieving a resultset from the resultset-detail
diff --git a/treeherder/model/derived/jobs.py b/treeherder/model/derived/jobs.py
index 9d5e46b81..940e4ad7f 100644
--- a/treeherder/model/derived/jobs.py
+++ b/treeherder/model/derived/jobs.py
@@ -66,8 +66,7 @@ class JobsModel(TreeherderModelBase):
             "revision_hash": "rs.revision_hash",
             "revision": "revision.revision",
             "author": "revision.author",
-            "push_timestamp": "rs.push_timestamp",
-            "revision_date": "revision.commit_timestamp"
+            "push_timestamp": "rs.push_timestamp"
         },
         "bug_job_map": {
             "job_id": "job_id",
@@ -589,27 +588,6 @@ class JobsModel(TreeherderModelBase):
         )
         return data
 
-    def get_push_timestamp_lookup_for_revisions(self, revisions):
-        """Get the push timestamp for a list of revisions."""
-
-        # Generate a list of result_set_ids
-        id_placeholders = []
-        repl = []
-        for data in result_set_ids:
-            id_placeholders.append('%s')
-        repl.append(','.join(id_placeholders))
-
-        proc = "jobs.selects.get_result_set_push_timestamp"
-        data = self.get_jobs_dhub().execute(
-            proc=proc,
-            placeholders=result_set_ids,
-            debug_show=self.DEBUG,
-            replace=repl,
-            return_type="dict",
-            key_column="id"
-        )
-        return data
-
     ##################
     #
     # Objectstore functionality
diff --git a/treeherder/webapp/api/bug.py b/treeherder/webapp/api/bug.py
index 7b8869092..e98c2de93 100644
--- a/treeherder/webapp/api/bug.py
+++ b/treeherder/webapp/api/bug.py
@@ -1,3 +1,4 @@
+from _mysql_exceptions import IntegrityError
 from rest_framework import viewsets
 from rest_framework.response import Response
 
@@ -18,10 +19,18 @@ class BugJobMapViewSet(viewsets.ViewSet):
         job_id, bug_id = map(int, (request.DATA['job_id'],
                              request.DATA['bug_id']))
 
-        jm.insert_bug_job_map(job_id, bug_id,
-                              request.DATA['type'])
+        resp_msg = "Bug job map saved"
+        try:
+            jm.insert_bug_job_map(job_id, bug_id,
+                                  request.DATA['type'])
+        except IntegrityError as e:
+            code, msg = e.args
+            if "Duplicate" in msg:
+                resp_msg = "Bug job map skipped: {0}".format(msg)
+            else:
+                raise e
 
-        return Response({"message": "Bug job map stored"})
+        return Response({"message": resp_msg})
 
     @with_jobs
     def destroy(self, request, project, jm, pk=None):
diff --git a/treeherder/webapp/api/resultset.py b/treeherder/webapp/api/resultset.py
index 891287a14..021cfd264 100644
--- a/treeherder/webapp/api/resultset.py
+++ b/treeherder/webapp/api/resultset.py
@@ -1,6 +1,4 @@
 import itertools
-import time
-import datetime
 
 from rest_framework import viewsets
 from rest_framework.response import Response
@@ -8,7 +6,9 @@ from rest_framework.decorators import link
 from rest_framework.reverse import reverse
 from treeherder.model.derived import DatasetNotFoundError
 from treeherder.webapp.api.utils import (UrlQueryFilter, with_jobs,
-                                         oauth_required, get_option)
+                                         oauth_required, get_option,
+                                         get_revision_timestamp,
+                                         to_timestamp)
 
 
 class ResultSetViewSet(viewsets.ViewSet):
@@ -24,18 +24,37 @@ class ResultSetViewSet(viewsets.ViewSet):
         GET method for list of ``resultset`` records with revisions
 
         """
-        def xlate_date(datestr):
-            # exp a date like 2014-03-31.  change to timestamp
-            return time.mktime(datetime.datetime.strptime(datestr, "%Y-%m-%d").timetuple())
 
-        filter = UrlQueryFilter(request.QUERY_PARAMS, {
-            "push_timestamp": xlate_date,
-        })
+        # make a mutable copy of these params
+        query_params = request.QUERY_PARAMS.copy()
 
-        # translate range filter conditions for date and revision
+        # support ranges for date as well as revisions(changes) like old tbpl
+        fromchange = query_params.pop("fromchange", None)
+        tochange = query_params.pop("tochange", None)
+        startdate = query_params.pop("startdate", None)
+        enddate = query_params.pop("enddate", None)
 
-        # fromchange = filter.pop("fromchange", None)
-        # tochange = filter.pop("tochange", None)
+        # translate these params into our own filtering mechanism
+        if fromchange:
+            query_params.update({
+                "push_timestamp__gte": get_revision_timestamp(jm, fromchange[0])
+            })
+        if tochange:
+            query_params.update({
+                "push_timestamp__lte": get_revision_timestamp(jm, tochange[0])
+            })
+        if startdate:
+            query_params.update({
+                "push_timestamp__gte": to_timestamp(startdate[0])
+            })
+        if enddate:
+            # add a day because we aren't supplying a time, just a date.  So
+            # we're doing ``less than``, rather than ``less than or equal to``.
+            query_params.update({
+                "push_timestamp__lt": to_timestamp(enddate[0]) + 86400
+            })
+
+        filter = UrlQueryFilter(query_params)
 
         offset_id = filter.pop("id__lt", 0)
         count = filter.pop("count", 10)
diff --git a/treeherder/webapp/api/utils.py b/treeherder/webapp/api/utils.py
index 542dfa4b3..e55f17f5e 100644
--- a/treeherder/webapp/api/utils.py
+++ b/treeherder/webapp/api/utils.py
@@ -2,6 +2,8 @@ from collections import defaultdict
 
 import simplejson as json
 import oauth2 as oauth
+import time
+import datetime
 
 from django.conf import settings
 from rest_framework.response import Response
@@ -51,7 +53,7 @@ class UrlQueryFilter(object):
 
     splitter = "__"
 
-    def __init__(self, query_params, translators):
+    def __init__(self, query_params, translators=None):
         self.raw_params = query_params
         self.conditions = defaultdict(set)
         for k, v in self.raw_params.iteritems():
@@ -66,7 +68,7 @@ class UrlQueryFilter(object):
                 operator = "="
 
             # run the value through the translator for this field
-            if field in translators.keys():
+            if translators and field in translators.keys():
                 v = translators[field](v)
 
             self.conditions[field].add((self.operators[operator], v))
@@ -224,3 +226,16 @@ def get_option(obj, option_collections):
     else:
         return None
 
+
+def to_timestamp(datestr):
+    """get a timestamp from a datestr like 2014-03-31"""
+    return time.mktime(datetime.datetime.strptime(
+        datestr,
+        "%Y-%m-%d"
+        ).timetuple())
+
+def get_revision_timestamp(jm, rev):
+    """Get the push timestamp of the resultset for a revision"""
+    return jm.get_revision_resultset_lookup([rev])[rev][
+        "push_timestamp"
+        ]

From 95257c3040a0c71809395494aeb43feed44ba20e Mon Sep 17 00:00:00 2001
From: Cameron Dawson <cdawson@mozilla.com>
Date: Mon, 14 Apr 2014 13:23:11 -0700
Subject: [PATCH 4/7] add missing field to job

---
 treeherder/model/sql/jobs.json | 1 +
 1 file changed, 1 insertion(+)

diff --git a/treeherder/model/sql/jobs.json b/treeherder/model/sql/jobs.json
index 5c4766491..f4b1f1393 100644
--- a/treeherder/model/sql/jobs.json
+++ b/treeherder/model/sql/jobs.json
@@ -218,6 +218,7 @@
                     jg.`symbol` as job_group_symbol,
                     jg.`description` as job_group_description,
                     j.`who`,
+                    j.failure_classification_id,
                     j.`result_set_id`,
                     j.`result`,
                     j.`state`,

From 611ee5b7c0755c5ad5c84fbcf89e51e693e85b69 Mon Sep 17 00:00:00 2001
From: Cameron Dawson <cdawson@mozilla.com>
Date: Tue, 15 Apr 2014 10:43:51 -0700
Subject: [PATCH 5/7] fix socket.io fetchJobs method to use GUID

---
 requirements/pure.txt              |  1 +
 treeherder/webapp/api/resultset.py | 44 +++++++++++++++++++++---------
 treeherder/webapp/api/utils.py     |  6 +---
 3 files changed, 33 insertions(+), 18 deletions(-)

diff --git a/requirements/pure.txt b/requirements/pure.txt
index 5083db1b7..ebb03553a 100644
--- a/requirements/pure.txt
+++ b/requirements/pure.txt
@@ -9,6 +9,7 @@ carrot==0.10.7
 djangorestframework==2.3.12
 Unipath==1.0
 django-rest-swagger==0.1.11
+django-extensions==1.3.3
 
 django-cors-headers==0.11
 
diff --git a/treeherder/webapp/api/resultset.py b/treeherder/webapp/api/resultset.py
index 021cfd264..d9043a106 100644
--- a/treeherder/webapp/api/resultset.py
+++ b/treeherder/webapp/api/resultset.py
@@ -26,35 +26,45 @@ class ResultSetViewSet(viewsets.ViewSet):
         """
 
         # make a mutable copy of these params
-        query_params = request.QUERY_PARAMS.copy()
+        filter_params = request.QUERY_PARAMS.copy()
 
         # support ranges for date as well as revisions(changes) like old tbpl
-        fromchange = query_params.pop("fromchange", None)
-        tochange = query_params.pop("tochange", None)
-        startdate = query_params.pop("startdate", None)
-        enddate = query_params.pop("enddate", None)
+        fromchange = filter_params.pop("fromchange", None)
+        tochange = filter_params.pop("tochange", None)
+        startdate = filter_params.pop("startdate", None)
+        enddate = filter_params.pop("enddate", None)
+
+        # This will contain some meta data about the request and results
+        meta = {}
 
         # translate these params into our own filtering mechanism
         if fromchange:
-            query_params.update({
-                "push_timestamp__gte": get_revision_timestamp(jm, fromchange[0])
+            meta['fromchange'] = fromchange[0]
+            filter_params.update({
+                "push_timestamp__gte": get_revision_timestamp(jm, meta['fromchange'])
             })
         if tochange:
-            query_params.update({
-                "push_timestamp__lte": get_revision_timestamp(jm, tochange[0])
+            meta['tochange'] = tochange[0]
+            filter_params.update({
+                "push_timestamp__lte": get_revision_timestamp(jm, meta['tochange'])
             })
         if startdate:
-            query_params.update({
+            meta['startdate'] = startdate[0]
+            filter_params.update({
                 "push_timestamp__gte": to_timestamp(startdate[0])
             })
         if enddate:
+            meta['enddate'] = enddate[0]
+
             # add a day because we aren't supplying a time, just a date.  So
             # we're doing ``less than``, rather than ``less than or equal to``.
-            query_params.update({
+            filter_params.update({
                 "push_timestamp__lt": to_timestamp(enddate[0]) + 86400
             })
 
-        filter = UrlQueryFilter(query_params)
+        meta['filter_params'] = filter_params
+
+        filter = UrlQueryFilter(filter_params)
 
         offset_id = filter.pop("id__lt", 0)
         count = filter.pop("count", 10)
@@ -66,7 +76,15 @@ class ResultSetViewSet(viewsets.ViewSet):
             full,
             filter.conditions
         )
-        return Response(self.get_resultsets_with_jobs(jm, objs, full, {}))
+
+        results = self.get_resultsets_with_jobs(jm, objs, full, {})
+        meta['count'] = len(results)
+        meta['repository'] = project
+
+        return Response({
+            'meta': meta,
+            'results': results
+        })
 
     @with_jobs
     def retrieve(self, request, project, jm, pk=None):
diff --git a/treeherder/webapp/api/utils.py b/treeherder/webapp/api/utils.py
index e55f17f5e..f3d3393b0 100644
--- a/treeherder/webapp/api/utils.py
+++ b/treeherder/webapp/api/utils.py
@@ -53,7 +53,7 @@ class UrlQueryFilter(object):
 
     splitter = "__"
 
-    def __init__(self, query_params, translators=None):
+    def __init__(self, query_params):
         self.raw_params = query_params
         self.conditions = defaultdict(set)
         for k, v in self.raw_params.iteritems():
@@ -67,10 +67,6 @@ class UrlQueryFilter(object):
                 field = k
                 operator = "="
 
-            # run the value through the translator for this field
-            if translators and field in translators.keys():
-                v = translators[field](v)
-
             self.conditions[field].add((self.operators[operator], v))
 
     def get(self, key, default=None):

From d297109ed6c1009ee6510ce79976e6d2e316eb88 Mon Sep 17 00:00:00 2001
From: Cameron Dawson <cdawson@mozilla.com>
Date: Wed, 16 Apr 2014 09:45:47 -0700
Subject: [PATCH 6/7] fix unit tests for change in /resultsets/ api

---
 tests/webapp/api/test_resultset_api.py | 53 +++++++++++++++++++++-----
 1 file changed, 43 insertions(+), 10 deletions(-)

diff --git a/tests/webapp/api/test_resultset_api.py b/tests/webapp/api/test_resultset_api.py
index 29706550d..97866b779 100644
--- a/tests/webapp/api/test_resultset_api.py
+++ b/tests/webapp/api/test_resultset_api.py
@@ -15,11 +15,13 @@ def test_resultset_list(webapp, eleven_jobs_processed, jm):
         reverse("resultset-list", kwargs={"project": jm.project})
     )
 
-    assert resp.status_int == 200
-    assert isinstance(resp.json, list)
-    rs_list = resp.json
+    results = resp.json['results']
+    meta = resp.json['meta']
 
-    assert len(rs_list) == 10
+    assert resp.status_int == 200
+    assert isinstance(results, list)
+
+    assert len(results) == 10
     exp_keys = set([
         u'id',
         u'repository_id',
@@ -34,9 +36,16 @@ def test_resultset_list(webapp, eleven_jobs_processed, jm):
         u'job_counts',
         u'platforms'
     ])
-    for rs in rs_list:
+    for rs in results:
         assert set(rs.keys()) == exp_keys
 
+    assert(meta == {
+        u'count': 10,
+        u'filter_params': {},
+        u'repository':
+        u'test_treeherder'
+    })
+
 
 def test_resultset_list_bad_project(webapp, jm):
     """
@@ -64,7 +73,7 @@ def test_resultset_list_empty_rs_still_show(webapp, initial_data,
         reverse("resultset-list", kwargs={"project": jm.project}),
     )
     assert resp.status_int == 200
-    assert len(resp.json) == 10
+    assert len(resp.json['results']) == 10
 
 
 def test_resultset_list_filter_by_revision(webapp, eleven_jobs_processed, jm):
@@ -77,10 +86,22 @@ def test_resultset_list_filter_by_revision(webapp, eleven_jobs_processed, jm):
         {"fromchange": "21fb3eed1b5f", "tochange": "909f55c626a8"}
     )
     assert resp.status_int == 200
-    assert len(resp.json) == 4
-    assert set([rs["revision"] for rs in resp.json]) == set(
+    results = resp.json['results']
+    meta = resp.json['meta']
+    assert len(results) == 4
+    assert set([rs["revision"] for rs in results]) == set(
         ["909f55c626a8","71d49fee325a","bb57e9f67223","21fb3eed1b5f"]
         )
+    assert(meta == {
+        u'count': 4,
+        u'fromchange': u'21fb3eed1b5f',
+        u'filter_params': {
+            u'push_timestamp__gte': 1384363842,
+            u'push_timestamp__lte': 1384365942
+        },
+        u'repository': u'test_treeherder',
+        u'tochange': u'909f55c626a8'}
+    )
 
 
 def test_resultset_list_filter_by_date(webapp, initial_data,
@@ -101,10 +122,22 @@ def test_resultset_list_filter_by_date(webapp, initial_data,
         {"startdate": "2013-08-10", "enddate": "2013-08-13"}
     )
     assert resp.status_int == 200
-    assert len(resp.json) == 4
-    assert set([rs["revision"] for rs in resp.json]) == set(
+    results = resp.json['results']
+    meta = resp.json['meta']
+    assert len(results) == 4
+    assert set([rs["revision"] for rs in results]) == set(
         ["909f55c626a8","71d49fee325a","bb57e9f67223","668424578a0d"]
         )
+    assert(meta == {
+        u'count': 4,
+        u'enddate': u'2013-08-13',
+        u'filter_params': {
+            u'push_timestamp__gte': 1376118000.0,
+            u'push_timestamp__lt': 1376463600.0
+        },
+        u'repository': u'test_treeherder',
+        u'startdate': u'2013-08-10'}
+    )
 
 
 def test_resultset_detail(webapp, eleven_jobs_processed, jm):

From d9527f7da5e48b59666f4de50d947bf769acf65e Mon Sep 17 00:00:00 2001
From: Cameron Dawson <cdawson@mozilla.com>
Date: Thu, 17 Apr 2014 13:28:54 -0700
Subject: [PATCH 7/7] updates per mdoglio's feedback

---
 requirements/dev.txt               |  1 +
 requirements/pure.txt              |  1 -
 treeherder/model/derived/jobs.py   | 35 ++++++++++++++++++++++--------
 treeherder/webapp/api/bug.py       | 12 +++++-----
 treeherder/webapp/api/resultset.py | 34 +++++++++++++----------------
 treeherder/webapp/api/utils.py     | 11 ++--------
 6 files changed, 51 insertions(+), 43 deletions(-)

diff --git a/requirements/dev.txt b/requirements/dev.txt
index 2c89ac398..8ed043a3c 100644
--- a/requirements/dev.txt
+++ b/requirements/dev.txt
@@ -13,6 +13,7 @@ WebTest==1.3.4
 WebOb==1.2
 
 mock==1.0b1
+django-extensions==1.3.3
 
 
 # in order to be able to run bin/generate-vendor-lib.py
diff --git a/requirements/pure.txt b/requirements/pure.txt
index ebb03553a..5083db1b7 100644
--- a/requirements/pure.txt
+++ b/requirements/pure.txt
@@ -9,7 +9,6 @@ carrot==0.10.7
 djangorestframework==2.3.12
 Unipath==1.0
 django-rest-swagger==0.1.11
-django-extensions==1.3.3
 
 django-cors-headers==0.11
 
diff --git a/treeherder/model/derived/jobs.py b/treeherder/model/derived/jobs.py
index 940e4ad7f..7a6675855 100644
--- a/treeherder/model/derived/jobs.py
+++ b/treeherder/model/derived/jobs.py
@@ -2,6 +2,8 @@ import json
 import MySQLdb
 import time
 
+from _mysql_exceptions import IntegrityError
+
 from warnings import filterwarnings, resetwarnings
 from django.conf import settings
 
@@ -316,15 +318,19 @@ class JobsModel(TreeherderModelBase):
         """
         Store a new relation between the given job and bug ids.
         """
-        self.get_jobs_dhub().execute(
-            proc='jobs.inserts.insert_bug_job_map',
-            placeholders=[
-                job_id,
-                bug_id,
-                assignment_type
-            ],
-            debug_show=self.DEBUG
-        )
+        try:
+            self.get_jobs_dhub().execute(
+                proc='jobs.inserts.insert_bug_job_map',
+                placeholders=[
+                    job_id,
+                    bug_id,
+                    assignment_type
+                ],
+                debug_show=self.DEBUG
+            )
+        except IntegrityError as e:
+            raise JobDataIntegrityError(e)
+
 
     def delete_bug_job_map(self, job_id, bug_id):
         """
@@ -1470,10 +1476,21 @@ class JobsModel(TreeherderModelBase):
             'revision_ids':revision_id_lookup
             }
 
+    def get_revision_timestamp(self, rev):
+        """Get the push timestamp of the resultset for a revision"""
+        return self.get_revision_resultset_lookup([rev])[rev][
+            "push_timestamp"
+            ]
+
+
 class JobDataError(ValueError):
     pass
 
 
+class JobDataIntegrityError(IntegrityError):
+    pass
+
+
 class JobData(dict):
     """
     Encapsulates data access from incoming test data structure.
diff --git a/treeherder/webapp/api/bug.py b/treeherder/webapp/api/bug.py
index e98c2de93..f27705caa 100644
--- a/treeherder/webapp/api/bug.py
+++ b/treeherder/webapp/api/bug.py
@@ -1,4 +1,4 @@
-from _mysql_exceptions import IntegrityError
+from treeherder.model.derived.jobs import JobDataIntegrityError
 from rest_framework import viewsets
 from rest_framework.response import Response
 
@@ -19,18 +19,20 @@ class BugJobMapViewSet(viewsets.ViewSet):
         job_id, bug_id = map(int, (request.DATA['job_id'],
                              request.DATA['bug_id']))
 
-        resp_msg = "Bug job map saved"
         try:
             jm.insert_bug_job_map(job_id, bug_id,
                                   request.DATA['type'])
-        except IntegrityError as e:
+        except JobDataIntegrityError as e:
             code, msg = e.args
             if "Duplicate" in msg:
-                resp_msg = "Bug job map skipped: {0}".format(msg)
+                return Response(
+                    {"message": "Bug job map skipped: {0}".format(msg)},
+                    409
+                )
             else:
                 raise e
 
-        return Response({"message": resp_msg})
+        return Response({"message": "Bug job map saved"})
 
     @with_jobs
     def destroy(self, request, project, jm, pk=None):
diff --git a/treeherder/webapp/api/resultset.py b/treeherder/webapp/api/resultset.py
index d9043a106..8d6b9aaee 100644
--- a/treeherder/webapp/api/resultset.py
+++ b/treeherder/webapp/api/resultset.py
@@ -7,7 +7,6 @@ from rest_framework.reverse import reverse
 from treeherder.model.derived import DatasetNotFoundError
 from treeherder.webapp.api.utils import (UrlQueryFilter, with_jobs,
                                          oauth_required, get_option,
-                                         get_revision_timestamp,
                                          to_timestamp)
 
 
@@ -28,38 +27,35 @@ class ResultSetViewSet(viewsets.ViewSet):
         # make a mutable copy of these params
         filter_params = request.QUERY_PARAMS.copy()
 
-        # support ranges for date as well as revisions(changes) like old tbpl
-        fromchange = filter_params.pop("fromchange", None)
-        tochange = filter_params.pop("tochange", None)
-        startdate = filter_params.pop("startdate", None)
-        enddate = filter_params.pop("enddate", None)
-
         # This will contain some meta data about the request and results
         meta = {}
 
+        # support ranges for date as well as revisions(changes) like old tbpl
+        for param in ["fromchange", "tochange", "startdate", "enddate"]:
+            v = filter_params.get(param, None)
+            if v:
+                del(filter_params[param])
+                meta[param] = v
+
         # translate these params into our own filtering mechanism
-        if fromchange:
-            meta['fromchange'] = fromchange[0]
+        if 'fromchange' in meta:
             filter_params.update({
-                "push_timestamp__gte": get_revision_timestamp(jm, meta['fromchange'])
+                "push_timestamp__gte": jm.get_revision_timestamp(meta['fromchange'])
             })
-        if tochange:
-            meta['tochange'] = tochange[0]
+        if 'tochange' in meta:
             filter_params.update({
-                "push_timestamp__lte": get_revision_timestamp(jm, meta['tochange'])
+                "push_timestamp__lte": jm.get_revision_timestamp(meta['tochange'])
             })
-        if startdate:
-            meta['startdate'] = startdate[0]
+        if 'startdate' in meta:
             filter_params.update({
-                "push_timestamp__gte": to_timestamp(startdate[0])
+                "push_timestamp__gte": to_timestamp(meta['startdate'])
             })
-        if enddate:
-            meta['enddate'] = enddate[0]
+        if 'enddate' in meta:
 
             # add a day because we aren't supplying a time, just a date.  So
             # we're doing ``less than``, rather than ``less than or equal to``.
             filter_params.update({
-                "push_timestamp__lt": to_timestamp(enddate[0]) + 86400
+                "push_timestamp__lt": to_timestamp(meta['enddate']) + 86400
             })
 
         meta['filter_params'] = filter_params
diff --git a/treeherder/webapp/api/utils.py b/treeherder/webapp/api/utils.py
index f3d3393b0..6cf134cbf 100644
--- a/treeherder/webapp/api/utils.py
+++ b/treeherder/webapp/api/utils.py
@@ -1,10 +1,9 @@
 from collections import defaultdict
-
-import simplejson as json
-import oauth2 as oauth
 import time
 import datetime
 
+import simplejson as json
+import oauth2 as oauth
 from django.conf import settings
 from rest_framework.response import Response
 
@@ -229,9 +228,3 @@ def to_timestamp(datestr):
         datestr,
         "%Y-%m-%d"
         ).timetuple())
-
-def get_revision_timestamp(jm, rev):
-    """Get the push timestamp of the resultset for a revision"""
-    return jm.get_revision_resultset_lookup([rev])[rev][
-        "push_timestamp"
-        ]