add dynamic filters to the jobs list endpoint

tests/webapp/api/test_jobs_api.py

@@ -137,6 +137,33 @@ def test_job_list_bad_project(webapp, eleven_jobs_processed, jm):
     webapp.get(badurl, status=404)
 
 
+def test_job_list_equals_filter(webapp, eleven_jobs_processed, jm):
+    """
+    test retrieving a job list with a querystring filter.
+    """
+    url = reverse("jobs-list",
+                  kwargs={"project": jm.project})
+    final_url = url + "?job_guid=f1c75261017c7c5ce3000931dce4c442fe0a1297"
+
+    resp = webapp.get(final_url)
+    assert len(resp.json) == 1
+
+
+def test_job_list_in_filter(webapp, eleven_jobs_processed, jm):
+    """
+    test retrieving a job list with a querystring filter.
+    """
+    url = reverse("jobs-list",
+                  kwargs={"project": jm.project})
+    final_url = url + ("?job_guid__in="
+    "f1c75261017c7c5ce3000931dce4c442fe0a1297,"
+    "9abb6f7d54a49d763c584926377f09835c5e1a32")
+
+    resp = webapp.get(final_url)
+    assert len(resp.json) == 2
+
+
+
 def test_job_detail(webapp, eleven_jobs_processed, sample_artifacts, jm):
     """
     test retrieving a single job from the jobs-detail

tests/webapp/api/test_utils.py

@@ -0,0 +1,42 @@
+from treeherder.webapp.api.utils import UrlQueryFilter
+
+
+def test_single_filter():
+    input = {
+        "name": "john",
+        "age__gte": 30,
+        "weight__lt": 80,
+        "gender__in": "male,female"
+    }
+
+    expected = {
+        'name': set([('=', 'john')]),
+        'age': set([('>=', 30)]),
+        'weight': set([('<', 80)]),
+        'gender': set([('IN', ("male", "female"))])
+    }
+
+    filter = UrlQueryFilter(input)
+    actual = filter.parse()
+
+    for k in expected:
+        assert actual[k] == expected[k]
+
+
+def test_multiple_filters():
+    input = {
+        "name": "john",
+        "age__gte": 30,
+        "age__lt": 80,
+    }
+
+    expected = {
+        'name': set([('=', 'john')]),
+        'age': set([('>=', 30), ('<', 80)]),
+    }
+
+    filter = UrlQueryFilter(input)
+    actual = filter.parse()
+
+    for k in expected:
+        assert actual[k] == expected[k]

treeherder/model/derived/jobs.py

@@ -27,6 +27,29 @@ class JobsModel(TreeherderModelBase):
     CT_OBJECTSTORE = "objectstore"
     CONTENT_TYPES = [CT_JOBS, CT_OBJECTSTORE]
     STATES = ["pending", "running", "completed", "coalesced"]
+    # list of searchable columns, i.e. those who have an index
+    # it would be nice to get this directly from the db and cache it
+    INDEXED_COLUMNS = {
+        "job": [
+            "job_guid",
+            "job_coalesced_to_guid",
+            "result_set_id",
+            "build_platform_id",
+            "machine_platform_id",
+            "machine_id",
+            "option_collection_hash",
+            "job_type_id",
+            "product_id",
+            "failure_classification_id",
+            "who",
+            "reason",
+            "result",
+            "state",
+            "submit_timestamp",
+            "start_timestamp",
+            "end_timestamp"
+        ]
+    }
 
     @classmethod
     def create(cls, project, host=None):
@@ -75,30 +98,46 @@ class JobsModel(TreeherderModelBase):
         )
         return data
 
-    def get_job_list(self, offset, limit, **kwargs):
+    def get_job_list(self, offset, limit, conditions=None):
         """
         Retrieve a list of jobs.
         Mainly used by the restful api to list the jobs
 
         joblist: a list of job ids to limit which jobs are returned.
         """
-        filter_str = ""
 
-        if "joblist" in kwargs:
-            filter_str += " AND j.id in ({0})".format(kwargs["joblist"])
+        placeholders = []
+        replace_str = ""
+        if conditions:
+            for column, condition in conditions.items():
+                if column in self.INDEXED_COLUMNS["job"]:
+                    for operator, value in condition:
+                        replace_str += "AND j.{0} {1}".format(column, operator)
+                        if operator == "IN":
+                            # create a list of placeholders of the same length
+                            # as the list of values
+                            replace_str += "({0})".format(
+                                ",".join(["%s"] * len(value))
+                            )
+                            placeholders += value
+                        else:
+                            replace_str += " %s "
+                            placeholders.append(value)
 
-        repl = [self.refdata_model.get_db_name(), filter_str]
+        repl = [self.refdata_model.get_db_name(), replace_str]
 
         proc = "jobs.selects.get_job_list"
+
         data = self.get_jobs_dhub().execute(
             proc=proc,
             replace=repl,
-            placeholders=[offset, limit],
+            placeholders=placeholders,
+            limit="{0},{1}".format(offset, limit),
             debug_show=self.DEBUG,
         )
-
         return data
 
+
     def set_state(self, job_id, state):
         """Update the state of an existing job"""
         self.get_jobs_dhub().execute(

treeherder/model/sql/jobs.json

@@ -158,7 +158,7 @@
         "update_last_job_classification":{
             "sql":"UPDATE `job`
                    SET `failure_classification_id` = ?
-                   WHERE `job_id` = ?",
+                   WHERE `id` = ?",
 
             "host":"master_host"
         }
@@ -247,8 +247,7 @@
 				  LEFT JOIN `REP0`.`job_group` as jg
                     ON jt.`job_group_id` = jg.id
                   WHERE 1
-                  REP1
-                  LIMIT ?,?",
+                  REP1",
 
             "host":"read_host"
         },

treeherder/webapp/api/utils.py

@@ -0,0 +1,62 @@
+from collections import defaultdict
+
+
+class UrlQueryFilter(object):
+    """
+    This class converts a set of querystring parameters
+    to a set of where conditions. It should be generic enough to
+    be used from any list method of a viewset. The style of filters
+    is strongly inspired by the django orm filters.
+
+    Examples of conversions:
+
+    {
+        "name": "john",
+        "age__gte":30,
+        "weight__lt":80
+        "gender__in": "male,female"
+    }
+
+    becomes
+
+    {
+        'name': set([('=', 'john')]),
+        'age': set([('>=', 30)]),
+        'weight': set([('<', 80)])
+        'gender': set([('IN', "male,female")])
+    }
+
+
+    """
+    operators = {
+        "gt": ">",
+        "gte": ">=",
+        "lt": "<",
+        "lte": "<=",
+        "=": "=",
+        "in": "IN"
+    }
+
+    splitter = "__"
+
+    def __init__(self, query_params):
+        self.params = query_params
+
+    def parse(self):
+        """
+        Parse the query_params using self.operators for the conversion
+        """
+        filters = defaultdict(set)
+        for k, v in self.params.iteritems():
+            if self.splitter in k:
+                field, operator = k.split(self.splitter, 1)
+                if operator not in self.operators:
+                    raise ValueError("{0} is not a supported operator".format(operator))
+                if operator == "in":
+                    v = tuple(v.split(","))
+            else:
+                field = k
+                operator = "="
+
+            filters[field].add((self.operators[operator], v))
+        return filters

treeherder/webapp/api/views.py

@@ -4,13 +4,14 @@ import itertools
 from django.conf import settings
 from rest_framework import viewsets
 from rest_framework.response import Response
-from rest_framework.decorators import action
+from rest_framework.decorators import action, link
 from rest_framework.reverse import reverse
 from rest_framework.exceptions import ParseError
 
 from treeherder.model import models
 from treeherder.model.derived import (JobsModel, DatasetNotFoundError,
                                       ObjectNotFoundException)
+from treeherder.webapp.api.utils import UrlQueryFilter
 
 
 def with_jobs(model_func):
@@ -203,18 +204,12 @@ class JobsViewSet(viewsets.ViewSet):
         GET method implementation for list view
 
         """
+        filters = UrlQueryFilter(request.QUERY_PARAMS).parse()
 
-        filters = ["joblist"]
+        limit_condition = filters.pop("limit", set([("=", "0,10")])).pop()
+        offset, limit = limit_condition[1].split(",")
+        objs = jm.get_job_list(offset, limit, filters)
 
-        offset = int(request.QUERY_PARAMS.get('offset', 0))
-        count = int(request.QUERY_PARAMS.get('count', 10))
-
-        objs = jm.get_job_list(
-            offset,
-            count,
-            **dict((k, v) for k, v in request.QUERY_PARAMS.iteritems()
-                   if k in filters)
-        )
         if objs:
             option_collections = jm.refdata_model.get_all_option_collections()
             for job in objs: