webapp/api test changes

tests/conftest.py

@@ -218,11 +218,15 @@ def mock_log_parser(monkeypatch):
 @pytest.fixture
 def result_set_stored(jm, initial_data, sample_resultset):
 
+    """
     jm.store_result_set_data(
         sample_resultset['revision_hash'],
         sample_resultset['push_timestamp'],
         sample_resultset['revisions']
     )
+    """
+    jm.store_result_set_data(sample_resultset)
+
     return sample_resultset
 
 
@@ -234,6 +238,26 @@ def mock_get_resultset(monkeypatch, result_set_stored):
     def _get_resultset(project, revision):
         return {
             'id': 1,
-            'revision_hash': result_set_stored['revision_hash']
+            'revision_hash': result_set_stored[0]['revision_hash']
         }
     monkeypatch.setattr(common, 'get_resultset', _get_resultset)
+
+@pytest.fixture()
+def refdata():
+    """returns a patched RefDataManager for testing purpose"""
+
+    import os
+    from treeherder.model.derived import RefDataManager
+    from tests.conftest import add_test_procs_file
+
+    refdata = RefDataManager()
+
+    proc_path = os.path.join(
+        os.path.abspath(os.path.dirname(__file__)),
+        'model',
+        'derived',
+        'test_refdata.json'
+    )
+
+    add_test_procs_file(refdata.dhub, 'reference', proc_path)
+    return refdata

tests/webapp/api/test_resultset_api.py

@@ -48,39 +48,19 @@ def test_resultset_list_bad_project(webapp, jm):
     assert resp.json == {"message": "No project with name foo"}
 
 
-def test_resultset_list_exclude_empty_no_rs(webapp, initial_data,
-                                            pushlog_sample, jm):
-    """
-    test retrieving a resultset list, when the resultset has no jobs.
-    should not show.
-    """
-    jm.store_result_set_data(pushlog_sample['revision_hash'],
-                             pushlog_sample['push_timestamp'],
-                             pushlog_sample['revisions'])
-
-    resp = webapp.get(
-        reverse("resultset-list", kwargs={"project": jm.project}),
-        {"exclude_empty": 1},
-    )
-    assert resp.status_int == 200
-    assert len(resp.json) == 0
-
-
 def test_resultset_list_empty_rs_still_show(webapp, initial_data,
                                             pushlog_sample, jm):
     """
     test retrieving a resultset list, when the resultset has no jobs.
     should not show.
     """
-    jm.store_result_set_data(pushlog_sample['revision_hash'],
-                             pushlog_sample['push_timestamp'],
-                             pushlog_sample['revisions'])
+    jm.store_result_set_data(pushlog_sample)
 
     resp = webapp.get(
         reverse("resultset-list", kwargs={"project": jm.project}),
     )
     assert resp.status_int == 200
-    assert len(resp.json) == 1
+    assert len(resp.json) == 10
 
 
 def test_resultset_detail(webapp, eleven_jobs_processed, jm):
@@ -275,25 +255,24 @@ def test_resultset_create(webapp, pushlog_sample, jm, initial_data):
 
     stored_objs = jm.get_jobs_dhub().execute(
         proc="jobs_test.selects.resultset_by_rev_hash",
-        placeholders=[pushlog_sample['revision_hash']]
+        placeholders=[pushlog_sample[0]['revision_hash']]
     )
 
     assert len(stored_objs) == 1
 
-    assert stored_objs[0]['revision_hash'] == pushlog_sample['revision_hash']
+    assert stored_objs[0]['revision_hash'] == pushlog_sample[0]['revision_hash']
 
 
 def test_result_set_add_job(jm, initial_data, webapp, job_sample, pushlog_sample):
 
-    jm.store_result_set_data(pushlog_sample['revision_hash'],
-                             pushlog_sample['push_timestamp'],
-                             pushlog_sample['revisions'])
+    jm.store_result_set_data(pushlog_sample)
 
-    job_sample['revision_hash'] = pushlog_sample['revision_hash']
+    job_sample['revision_hash'] = pushlog_sample[0]['revision_hash']
     job_sample['job']['log_references'] = []
+
     resp = webapp.post_json(
         reverse("resultset-add-job",
                 kwargs={"project": jm.project, "pk": 1}),
-        params=job_sample
+        params=[job_sample]
     )
     assert resp.status_int == 200

treeherder/etl/mixins.py

@@ -59,18 +59,31 @@ class JobsLoaderMixin(JsonLoaderMixin):
     def load(self, jobs):
         """post a list of jobs to the objectstore ingestion endpoint """
 
+        project_jobs_map = {}
+
         for job in jobs:
+
             project = job['project']
 
+            if project not in project_jobs_map:
+                project_jobs_map[project] = []
+
+            project_jobs_map[project].append(job)
+
+        for project in project_jobs_map:
+
             # the creation endpoint is the same as the list one
-            endpoint = reverse("resultset-add-job",
-                kwargs={"project": project, "pk": job['resultset_id']})
+            endpoint = reverse(
+                "resultset-add-job",
+                kwargs={"project": project, "pk":1 },
+                params=project_jobs_map[project]
+                )
 
             url = "{0}/{1}/".format(
                 settings.API_HOSTNAME.strip('/'),
                 endpoint.strip('/')
             )
-            response = super(JobsLoaderMixin, self).load(url, job)
+            response = super(JobsLoaderMixin, self).load(url)
 
             if response.getcode() != 200:
                 message = json.loads(response.read())

treeherder/model/derived/jobs.py

@@ -265,10 +265,6 @@ class JobsModel(TreeherderModelBase):
         # message the user, it would save us a very expensive join
         # with the jobs table.
 
-        #TODO: Remove if this is not necessary
-        if "exclude_empty" in kwargs and int(kwargs["exclude_empty"]) == 1:
-            replace_str += " AND job.id is not null"
-
         placeholders.extend([offset, limit])
 
         # Retrieve the filtered/limited list of result sets
@@ -281,7 +277,7 @@ class JobsModel(TreeherderModelBase):
         )
 
         aggregate_details = self.get_result_set_details(result_set_ids)
-        
+
         # Construct the return dataset, include all revisions associated
         # with each result_set in the revision_list attribute
         return_list = []
@@ -301,7 +297,7 @@ class JobsModel(TreeherderModelBase):
                     "revision_list":aggregate_details[ result['id'] ]
                     }
                 )
-                    
+
         return self.as_list(return_list, "result_set", **kwargs)
 
     def get_result_set_details(self, result_set_ids):
@@ -547,23 +543,41 @@ class JobsModel(TreeherderModelBase):
         # loaded. Used to mark the status complete.
         object_placeholders = []
 
+        print "DATA"
+        print data
         for datum in data:
             # Make sure we can deserialize the json object
             # without raising an exception
             try:
-                job_struct = JobData.from_json(datum['json_blob'])
-                revision_hash = job_struct['revision_hash']
-                job = job_struct['job']
+                if 'json_blob' in datum:
+                    job_struct = JobData.from_json(datum['json_blob'])
+                    revision_hash = job_struct['revision_hash']
+                    job = job_struct['job']
+                else:
+                    job = datum['job']
+                    revision_hash = datum['revision_hash']
+
+                # TODO: Need a job structure validation step here. Now that
+                # everything works in list context we cannot detect what
+                # object is responsible for what error. If we validate here
+                # we can capture the error and associate it with the object
+                # and also skip it before generating any database errors.
+
             except JobDataError as e:
-                self.mark_object_error(datum['id'], str(e))
+
+                if 'id' in datum:
+                    self.mark_object_error(datum['id'], str(e))
+
                 if raise_errors:
                     raise e
+
             except Exception as e:
-                self.mark_object_error(
-                    datum['id'],
-                    u"Unknown error: {0}: {1}".format(
-                        e.__class__.__name__, unicode(e))
-                )
+                if 'id' in datum:
+                    self.mark_object_error(
+                        datum['id'],
+                        u"Unknown error: {0}: {1}".format(
+                            e.__class__.__name__, unicode(e))
+                    )
                 if raise_errors:
                     raise e
             else:
@@ -581,9 +595,10 @@ class JobsModel(TreeherderModelBase):
                     artifact_placeholders
                     )
 
-                object_placeholders.append(
-                    [ revision_hash, datum['id'] ]
-                    )
+                if 'id' in datum:
+                    object_placeholders.append(
+                        [ revision_hash, datum['id'] ]
+                        )
 
         # Store all reference data and retrieve associated ids
         id_lookups = self.refdata_model.set_all_reference_data()
@@ -959,12 +974,13 @@ class JobsModel(TreeherderModelBase):
                 ...
                 ]
         """
-        self.get_os_dhub().execute(
-            proc="objectstore.updates.mark_complete",
-            placeholders=object_placeholders,
-            executemany=True,
-            debug_show=self.DEBUG
-        )
+        if object_placeholders:
+            self.get_os_dhub().execute(
+                proc="objectstore.updates.mark_complete",
+                placeholders=object_placeholders,
+                executemany=True,
+                debug_show=self.DEBUG
+            )
 
     def mark_object_error(self, object_id, error):
         """ Call to database to mark the task completed """

treeherder/model/sql/jobs.json

@@ -239,8 +239,6 @@
                       rs.revision_hash,
                       rs.push_timestamp
                    FROM result_set AS rs
-                   LEFT JOIN job
-                      ON job.result_set_id = rs.id
                    INNER JOIN revision_map
                       ON rs.id = revision_map.result_set_id
                    INNER JOIN revision

treeherder/model/sql/reference.json

@@ -238,9 +238,9 @@
                         GROUP_CONCAT( name SEPARATOR ' ' ) as opt
                    FROM `option_collection` oc
                    INNER JOIN `option` o
-                        on o.id = oc.option_id
-                    GROUP BY oc.option_collection_hash
-                   WHERE o.active_statue = 'active'",
+                        ON o.id = oc.option_id
+                   WHERE o.active_status = 'active'
+                   GROUP BY oc.option_collection_hash",
             "host":"read_host"
         }
     },

treeherder/webapp/api/views.py

@@ -349,12 +349,7 @@ class ResultSetViewSet(viewsets.ViewSet):
         The incoming data has the same structure as for
         the objectstore ingestion.
         """
-
-        job = request.DATA
-
-        jm.load_job_data(
-            job
-        )
+        jm.load_job_data(request.DATA)
 
         return Response({'message': 'Job successfully updated'})