Bug 1140349 - Skip and drain the objectstore

Since we use Celery for queueing job ingestion, the objectstore is
now irrelevant.  This code is the first step.  This will bypass
the Objectstore and ingest jobs directly to our ``jobs`` database.

Phase 2 is to remove all the Objectstore code (in a later commit)

Phase 3 is to delete the Objectstore databases and related fields in
other tables.

tests/client/test_treeherder_client.py

@@ -293,10 +293,7 @@ class TreeherderJobCollectionTest(DataSetup, unittest.TestCase):
 
         tjc = TreeherderJobCollection()
 
-        tjc_update = TreeherderJobCollection(job_type='update')
-
-        self.assertTrue(tjc.endpoint_base, 'objectstore')
-        self.assertTrue(tjc_update.endpoint_base, 'jobs')
+        self.assertTrue(tjc.endpoint_base, 'jobs')
 
 
 class TreeherderJobTest(DataSetup, unittest.TestCase):
@@ -514,7 +511,7 @@ class TreeherderClientTest(DataSetup, unittest.TestCase):
         self.assertEqual(mock_post.call_count, 1)
 
         path, resp = mock_post.call_args
-        self.assertEqual(path[0], "http://host/api/project/project/objectstore/?oauth_body_hash=IKbDoi5GvTRaqjRTCDyKIN5wWiY%3D&oauth_nonce=46810593&oauth_timestamp=1342229050&oauth_consumer_key=key&oauth_signature_method=HMAC-SHA1&oauth_version=1.0&oauth_token=&user=project&oauth_signature=uq%2BrkJCRPyPUdXExSasm25ab8m4%3D")
+        self.assertEqual(path[0], "http://host/api/project/project/jobs/?oauth_body_hash=IKbDoi5GvTRaqjRTCDyKIN5wWiY%3D&oauth_nonce=46810593&oauth_timestamp=1342229050&oauth_consumer_key=key&oauth_signature_method=HMAC-SHA1&oauth_version=1.0&oauth_token=&user=project&oauth_signature=DJe%2F%2FJtw7s2XUrciG%2Bl1tfJJen8%3D")
 
 if __name__ == '__main__':
     unittest.main()

tests/e2e/test_client_job_ingestion.py

@@ -49,7 +49,6 @@ def check_artifacts(test_project,
                     exp_error_summary=None):
 
     with JobsModel(test_project) as jobs_model:
-        jobs_model.process_objects(10)
         job_id = [x['id'] for x in jobs_model.get_job_list(0, 20)
                   if x['job_guid'] == job_guid][0]
         job_log_list = jobs_model.get_job_log_url_list([job_id])
@@ -415,7 +414,6 @@ def test_post_job_with_tier(test_project, result_set_stored,
     do_post_collection(test_project, tjc)
 
     with JobsModel(test_project) as jobs_model:
-        jobs_model.process_objects(10)
         job = [x for x in jobs_model.get_job_list(0, 20)
                if x['job_guid'] == job_guid][0]
         assert job['tier'] == 3
@@ -440,7 +438,33 @@ def test_post_job_with_default_tier(test_project, result_set_stored,
     do_post_collection(test_project, tjc)
 
     with JobsModel(test_project) as jobs_model:
-        jobs_model.process_objects(10)
         job = [x for x in jobs_model.get_job_list(0, 20)
                if x['job_guid'] == job_guid][0]
         assert job['tier'] == 1
+
+
+def test_post_job_to_deprecated_os_endpoint(test_project, result_set_stored,
+                                            mock_post_collection):
+    """
+    test submitting a job to deprecated objectstore endpoint
+    """
+
+    tjc = client.TreeherderJobCollection()
+    job_guid = 'd22c74d4aa6d2a1dcba96d95dccbd5fdca70cf33'
+    tj = client.TreeherderJob({
+        'project': test_project,
+        'revision_hash': result_set_stored[0]['revision_hash'],
+        'job': {
+            'job_guid': job_guid,
+            'state': 'completed',
+        }
+    })
+    tjc.add(tj)
+    tjc.endpoint_base = 'objectstore'
+
+    do_post_collection(test_project, tjc)
+
+    with JobsModel(test_project) as jobs_model:
+        job = [x for x in jobs_model.get_job_list(0, 20)
+               if x['job_guid'] == job_guid]
+        assert len(job) == 1

tests/etl/test_buildapi.py

@@ -155,14 +155,13 @@ def test_ingest_builds4h_jobs(jm, initial_data,
     from treeherder.etl.buildapi import Builds4hJobsProcess
     etl_process = Builds4hJobsProcess()
     etl_process.run()
-    jm.process_objects(20)
 
     stored_obj = jm.get_jobs_dhub().execute(
         proc="jobs_test.selects.jobs")
 
     jm.disconnect()
 
-    assert len(stored_obj) == 20
+    assert len(stored_obj) == 32
 
 
 def test_ingest_running_to_complete_job(jm, initial_data,
@@ -175,7 +174,6 @@ def test_ingest_running_to_complete_job(jm, initial_data,
     """
     a new buildapi running job transitions to a new completed job
 
-    Also ensure that a running job does NOT go through the objectstore.
     """
     from treeherder.etl.buildapi import RunningJobsProcess
     from treeherder.etl.buildapi import Builds4hJobsProcess
@@ -186,26 +184,19 @@ def test_ingest_running_to_complete_job(jm, initial_data,
     stored_running = jm.get_jobs_dhub().execute(
         proc="jobs_test.selects.jobs")
 
-    stored_objectstore = jm.get_os_dhub().execute(
-        proc="objectstore_test.selects.all")
-
-    # ensure running jobs do not go to the objectstore, but go directly
-    # to the jobs table without needing process_objects
-    assert len(stored_objectstore) == 0
     assert len(stored_running) == 1
 
     # the first job in the sample data should overwrite the running job
-    # we just ingested.  Leaving us with only 20 jobs, not 21.
+    # we just ingested.  Leaving us with only 32 jobs, not 33.
     etl_process = Builds4hJobsProcess()
     etl_process.run()
-    jm.process_objects(20)
 
     stored_obj = jm.get_jobs_dhub().execute(
         proc="jobs_test.selects.jobs")
 
     jm.disconnect()
 
-    assert len(stored_obj) == 20
+    assert len(stored_obj) == 32
 
     # all jobs should be completed, including the original one which
     # transitioned from running.
@@ -286,7 +277,6 @@ def test_ingest_builds4h_jobs_missing_branch(jm, initial_data,
     etl_process = Builds4hJobsProcess()
 
     etl_process.run()
-    jm.process_objects(2)
 
     stored_obj = jm.get_jobs_dhub().execute(
         proc="jobs_test.selects.jobs")
@@ -318,7 +308,6 @@ def _do_missing_resultset_test(jm, etl_process):
                   content_type='application/json')
 
     etl_process.run()
-    jm.process_objects(2)
 
     stored_obj = jm.get_jobs_dhub().execute(
         proc="jobs_test.selects.jobs")

tests/webapp/api/test_objectstore_api.py

@@ -9,36 +9,6 @@ from treeherder.client import TreeherderJobCollection
 from tests import test_utils
 
 
-def test_objectstore_create(job_sample, jm):
-    """
-    test posting data to the objectstore via webtest.
-    extected result are:
-    - return code 200
-    - return message successful
-    - 1 job stored in the objectstore
-    """
-
-    tjc = TreeherderJobCollection()
-    tj = tjc.get_job(job_sample)
-    tjc.add(tj)
-
-    resp = test_utils.post_collection(jm.project, tjc)
-
-    assert resp.status_int == 200
-    assert resp.json['message'] == 'well-formed JSON stored'
-
-    stored_objs = jm.get_os_dhub().execute(
-        proc="objectstore_test.selects.row_by_guid",
-        placeholders=[job_sample["job"]["job_guid"]]
-    )
-
-    assert len(stored_objs) == 1
-
-    assert stored_objs[0]['job_guid'] == job_sample["job"]["job_guid"]
-
-    jm.disconnect()
-
-
 def test_objectstore_list(webapp, eleven_jobs_stored, jm):
     """
     test retrieving a list of ten json blobs from the objectstore-list

treeherder/client/thclient/client.py

@@ -560,12 +560,7 @@ class TreeherderJobCollection(TreeherderCollection):
 
     def __init__(self, data=[], job_type=''):
 
-        if job_type == 'update':
-            endpoint_base = 'jobs'
-        else:
-            endpoint_base = 'objectstore'
-
-        super(TreeherderJobCollection, self).__init__(endpoint_base, data)
+        super(TreeherderJobCollection, self).__init__('jobs', data)
 
     def get_job(self, data={}):
 

treeherder/webapp/api/objectstore.py

@@ -24,19 +24,22 @@ class ObjectstoreViewSet(viewsets.ViewSet):
     @oauth_required
     def create(self, request, project, jm):
         """
-        POST method implementation
+        ::DEPRECATED:: POST method implementation
+
+        TODO: This can be removed when no more clients are using this endpoint.
+        Can verify with New Relic
+
+        This copies the exact implementation from the
+        /jobs/ create endpoint for backward compatibility with previous
+        versions of the Treeherder client and api.
         """
-        job_errors_resp = jm.store_job_data(request.DATA)
+        jm.load_job_data(request.DATA)
 
-        resp = {}
-        if job_errors_resp:
-            resp['message'] = job_errors_resp
-            status = 500
-        else:
-            status = 200
-            resp['message'] = 'well-formed JSON stored'
-
-        return Response(resp, status=status)
+        return Response('DEPRECATED: {}  {}: {}'.format(
+            "This API will be removed soon.",
+            "Please change to using",
+            "/api/project/{}/jobs/".format(project)
+        ))
 
     @with_jobs
     def retrieve(self, request, project, jm, pk=None):