Bug 1163674 - Update treeherder client to be more generic

* Create a generic TreeherderClient class
* Add a single method called `post_collection` which takes care of all
  details of validation, submitting stuff and raising errors
* Also add a new update_parse_status method, for updating status (replaces
  manual calls to post information on raw TreeherderRequest)

docs/submitting_data.rst

@@ -172,7 +172,7 @@ data structures to send, do something like this.
 
 .. code-block:: python
 
-    from thclient import TreeherderRequest, TreeherderResultSetCollection, TreeherderClientError
+    from thclient import TreeherderClient, TreeherderResultSetCollection, TreeherderClientError
 
     trsc = TreeherderResultSetCollection()
 
@@ -203,26 +203,20 @@ data structures to send, do something like this.
 
     # The OAuth key and secret for your project should be supplied to you by the
     # treeherder-service administrator.
-    req = TreeherderRequest(
-        protocol='https',
-        host='treeherder.mozilla.org',
-        project='project',
-        oauth_key='oauth-key',
-        oauth_secret='oauth-secret',
-        )
+    client = TreeherderClient(protocol='https', host='treeherder.mozilla.org')
 
-    # Post the result collection
+    # Post the result collection to a project
     #
     # data structure validation is automatically performed here, if validation
     # fails a TreeherderClientError is raised
-    req.post(trc)
+    client.post_collection('mozilla-central', 'oauth_key', 'oauth_secret', trc)
 
 At any time in building a data structure, you can examine what has been
 created by looking at the `data` property.  You can also call the `validate`
 method at any time before sending a collection.  All treeherder data classes
 have `validate` methods that can be used for testing.  The `validate` method
-is called on every structure in a collection when a `send` is called on a
-`TreeherderRequest`.  If validation fails a `TreeherderClientError` is raised.
+is called on every structure in a collection when `post_collection` is
+called. If validation fails a `TreeherderClientError` is raised.
 
 If you want to use `TreeherderJobCollection` to build up the job data
 structures to send, do something like this:
@@ -284,27 +278,15 @@ structures to send, do something like this:
                 )
         tjc.add(tj)
 
-    # Send the collection to treeherder
-    req = TreeherderRequest(
-        protocol='https',
-        host='treeherder.mozilla.org',
-        project='project',
-        oauth_key='oauth-key',
-        oauth_secret='oauth-secret',
-        )
-
-    # Post the job collection
-    #
-    # data structure validation is automatically performed here, if validation
-    # fails a TreeherderClientError is raised
-    req.post(tjc)
+    client = TreeherderClient(protocol='https', host='treeherder.mozilla.org')
+    client.post_collection('mozilla-central', 'oauth_key', 'oauth_secret', tjc)
 
 If you want to use `TreeherderArtifactCollection` to build up the job
 artifacts data structures to send, do something like this:
 
 .. code-block:: python
 
-    from thclient import TreeherderRequest, TreeherderArtifactCollection, TreeherderClientError
+    from thclient import TreeherderClient, TreeherderArtifactCollection, TreeherderClientError
 
     tac = TreeherderArtifactCollection()
 
@@ -320,19 +302,8 @@ artifacts data structures to send, do something like this:
         tac.add(ta)
 
     # Send the collection to treeherder
-    req = TreeherderRequest(
-        protocol='https',
-        host='treeherder.mozilla.org',
-        project='project',
-        oauth_key='oauth-key',
-        oauth_secret='oauth-secret',
-        )
-
-    # Post the artifact collection
-    #
-    # data structure validation is automatically performed here, if validation
-    # fails a TreeherderClientError is raised
-    req.post(tac)
+    client = TreeherderClient(protocol='https', host='treeherder.mozilla.org')
+    client.post_collection('mozilla-central', 'oauth_key', 'oauth_secret', tac)
 
 If you don't want to use `TreeherderResultCollection` or
 `TreeherderJobCollection` to build up the data structure to send, build the
@@ -352,16 +323,10 @@ data structures directly and add them to the collection.
         # add resultset to collection
         trc.add(trs)
 
-    req = TreeherderRequest(
-        protocol='https',
-        host='treeherder.mozilla.org',
-        project='project',
-        oauth_key='oauth-key',
-        oauth_secret='oauth-secret',
-        )
+    client = TreeherderClient(protocol='https', host='treeherder.mozilla.org')
+    client.post_collection('mozilla-central', 'oauth_key', 'oauth_secret', trc)
 
-    # Post the request to treeherder
-    req.post(trc)
+.. code-block:: python
 
     from thclient import TreeherderRequest, TreeherderJobCollection
 
@@ -375,19 +340,8 @@ data structures directly and add them to the collection.
         # add job to collection
         tjc.add(tj)
 
-    req = TreeherderRequest(
-        protocol='https',
-        host='treeherder.mozilla.org',
-        project='project',
-        oauth_key='oauth-key',
-        oauth_secret='oauth-secret',
-        )
-
-    # Post the request to treeherder
-    #
-    # data structure validation is automatically performed here, if validation
-    # fails a TreeherderClientError is raised
-    req.post(tjc)
+    client = TreeherderClient(protocol='https', host='treeherder.mozilla.org')
+    client.post_collection('mozilla-central', 'oauth_key', 'oauth_secret', tjc)
 
 In the same way, if you don't want to use `TreeherderArtifactCollection` to
 build up the data structure to send, build the data structures directly and
@@ -407,16 +361,8 @@ add them to the collection.
         # add artifact to collection
         tac.add(ta)
 
-    req = TreeherderRequest(
-        protocol='https',
-        host='treeherder.mozilla.org',
-        project='project',
-        oauth_key='oauth-key',
-        oauth_secret='oauth-secret',
-        )
-
-    # Post the request to treeherder
-    req.post(tac)
+    client = TreeherderClient(protocol='https', host='treeherder.mozilla.org')
+    client.post_collection('mozilla-central', 'oauth_key', 'oauth_secret', tac)
 
 Job artifacts format
 --------------------

tests/client/test_treeherder_client.py

@@ -13,7 +13,7 @@ from mock import patch
 from treeherder.client import (TreeherderJob, TreeherderJobCollection,
                                TreeherderRevision, TreeherderResultSet,
                                TreeherderResultSetCollection,
-                               TreeherderClientError, TreeherderRequest,
+                               TreeherderClient, TreeherderClientError,
                                TreeherderArtifact,
                                TreeherderArtifactCollection)
 
@@ -373,11 +373,20 @@ class TreeherderJobTest(DataSetup, unittest.TestCase):
                           'foo', 'bar', 'baz')
 
 
-class TreeherderRequestTest(DataSetup, unittest.TestCase):
+class TreeherderClientTest(DataSetup, unittest.TestCase):
 
-    @patch.object(TreeherderRequest, 'send')
-    def test_send_job_collection(self, mock_send):
+    @staticmethod
+    def _expected_response_return_object():
+        class Mock(object):
+            pass
+        ret = Mock()
+        setattr(ret, 'raise_for_status', lambda: None)
+        return ret
+
+    @patch("treeherder.client.client.requests.post")
+    def test_post_job_collection(self, mock_post):
         """Can add a treeherder collections to a TreeherderRequest."""
+        mock_post.return_value = self._expected_response_return_object()
 
         tjc = TreeherderJobCollection()
 
@@ -385,25 +394,25 @@ class TreeherderRequestTest(DataSetup, unittest.TestCase):
 
             tjc.add(tjc.get_job(job))
 
-        req = TreeherderRequest(
+        client = TreeherderClient(
             protocol='http',
             host='host',
-            project='project',
-            oauth_key='key',
-            oauth_secret='secret',
             )
 
-        req.post(tjc)
+        client.post_collection('project', 'key', 'secret', tjc)
 
-        self.assertEqual(mock_send.call_count, 1)
+        path, resp = mock_post.call_args
+
+        self.assertEqual(mock_post.call_count, 1)
         self.assertEqual(
             tjc.to_json(),
-            mock_send.call_args_list[0][1]['data']
+            resp['data']
             )
 
-    @patch.object(TreeherderRequest, 'send')
-    def test_send_result_collection(self, mock_send):
+    @patch("treeherder.client.client.requests.post")
+    def test_send_result_collection(self, mock_post):
         """Can add a treeherder collections to a TreeherderRequest."""
+        mock_post.return_value = self._expected_response_return_object()
 
         trc = TreeherderResultSetCollection()
 
@@ -411,25 +420,25 @@ class TreeherderRequestTest(DataSetup, unittest.TestCase):
 
             trc.add(trc.get_resultset(resultset))
 
-        req = TreeherderRequest(
+        client = TreeherderClient(
             protocol='http',
             host='host',
-            project='project',
-            oauth_key='key',
-            oauth_secret='secret',
             )
 
-        req.post(trc)
+        client.post_collection('project', 'key', 'secret', trc)
 
-        self.assertEqual(mock_send.call_count, 1)
+        path, resp = mock_post.call_args
+
+        self.assertEqual(mock_post.call_count, 1)
         self.assertEqual(
             trc.to_json(),
-            mock_send.call_args_list[0][1]['data']
+            resp['data']
             )
 
-    @patch.object(TreeherderRequest, 'send')
-    def test_send_artifact_collection(self, mock_send):
+    @patch("treeherder.client.client.requests.post")
+    def test_send_artifact_collection(self, mock_post):
         """Can add a artifact collections to a TreeherderRequest."""
+        mock_post.return_value = self._expected_response_return_object()
 
         tac = TreeherderArtifactCollection()
 
@@ -437,86 +446,37 @@ class TreeherderRequestTest(DataSetup, unittest.TestCase):
 
             tac.add(tac.get_artifact(artifact))
 
-        req = TreeherderRequest(
+        client = TreeherderClient(
             protocol='http',
             host='host',
-            project='project',
-            oauth_key='key',
-            oauth_secret='secret',
-        )
-
-        req.post(tac)
-
-        self.assertEqual(mock_send.call_count, 1)
-        self.assertEqual(
-            tac.to_json(),
-            mock_send.call_args_list[0][1]["data"]
-        )
-
-    @patch("treeherder.client.client.oauth.generate_nonce")
-    @patch("treeherder.client.client.oauth.time.time")
-    @patch("treeherder.client.client.requests.post")
-    def test_send(self, mock_post, mock_time, mock_generate_nonce):
-
-        """Can send data to the server."""
-        mock_time.return_value = 1342229050
-        mock_generate_nonce.return_value = "46810593"
-
-        host = 'host'
-
-        req = TreeherderRequest(
-            protocol='http',
-            host=host,
-            project='project',
-            oauth_key='key',
-            oauth_secret='secret',
             )
 
-        mock_response = mock_post.return_value
-
-        tjc = TreeherderJobCollection()
-
-        tjc.add(tjc.get_job(self.job_data[0]))
-
-        response = req.post(tjc)
-
-        self.assertEqual(mock_response, response)
-        self.assertEqual(mock_post.call_count, 1)
+        client.post_collection('project', 'key', 'secret', tac)
 
         path, resp = mock_post.call_args
 
-        deserialized_data = json.loads(resp['data'])
+        self.assertEqual(mock_post.call_count, 1)
         self.assertEqual(
-            deserialized_data,
-            tjc.get_collection_data()
-            )
-        self.assertEqual(
-            resp['headers']['Content-Type'],
-            'application/json',
+            tac.to_json(),
+            resp['data']
             )
 
     @patch("treeherder.client.client.oauth.generate_nonce")
     @patch("treeherder.client.client.oauth.time.time")
     @patch("treeherder.client.client.requests.post")
-    def test_send_without_oauth(self, mock_post, mock_time,
-                                mock_generate_nonce):
+    def test_send_with_oauth(self, mock_post, mock_time,
+                             mock_generate_nonce):
 
-        """Can send data to the server."""
+        """Tests that oauth data is sent to server"""
         mock_time.return_value = 1342229050
         mock_generate_nonce.return_value = "46810593"
+        mock_post.return_value = self._expected_response_return_object()
 
-        host = 'host'
-
-        req = TreeherderRequest(
+        client = TreeherderClient(
             protocol='http',
-            host=host,
-            project='project',
-            oauth_key=None,
-            oauth_secret=None,
+            host='host',
             )
 
-        mock_response = mock_post.return_value
-
         tjc = TreeherderJobCollection()
 
         for job in self.job_data:
@@ -524,22 +484,12 @@ class TreeherderRequestTest(DataSetup, unittest.TestCase):
             tjc.add(tjc.get_job(job))
             break
 
-        response = req.post(tjc)
+        client.post_collection('project', 'key', 'secret', tjc)
 
-        self.assertEqual(mock_response, response)
         self.assertEqual(mock_post.call_count, 1)
 
         path, resp = mock_post.call_args
-
-        deserialized_data = json.loads(resp['data'])
-        self.assertEqual(
-            deserialized_data,
-            tjc.get_collection_data()
-            )
-        self.assertEqual(
-            resp['headers']['Content-Type'],
-            'application/json',
-            )
+        self.assertEqual(path[0], "http://host/api/project/project/objectstore/?oauth_body_hash=C4jFXK8TBoFeh9wHOu1IkU7tERw%3D&oauth_nonce=46810593&oauth_timestamp=1342229050&oauth_consumer_key=key&oauth_signature_method=HMAC-SHA1&oauth_version=1.0&oauth_token=&user=project&oauth_signature=hNqHsAd7sdGyDLfWf7n9Bb%2B2rzM%3D")
 
 if __name__ == '__main__':
     unittest.main()

tests/conftest.py

@@ -5,13 +5,14 @@
 import os
 from os.path import dirname
 import sys
-import json
 
+import json
 import kombu
 import pytest
 from django.core.management import call_command
 from webtest.app import TestApp
 import responses
+import time
 
 from tests.sampledata import SampleData
 from treeherder.etl.oauth_utils import OAuthCredentials
@@ -378,27 +379,47 @@ def set_oauth_credentials():
 
 
 @pytest.fixture
-def mock_send_request(monkeypatch, set_oauth_credentials):
-    def _send(th_request, endpoint, method=None, data=None):
+def mock_post_collection(monkeypatch, set_oauth_credentials):
+    def _post_collection(th_client, project, oauth_key, oauth_secret,
+                         collection_inst, timeout=None):
 
-        if data and not isinstance(data, str):
-            data = json.dumps(data)
-
-        signed_uri = th_request.oauth_client.get_signed_uri(
-            data, th_request.get_uri(endpoint), method
-        )
-
-        response = getattr(TestApp(application), method.lower())(
-            str(signed_uri),
-            params=data,
+        jsondata = collection_inst.to_json()
+        signed_uri = th_client._get_uri(project, collection_inst.endpoint_base,
+                                        data=jsondata, oauth_key=oauth_key,
+                                        oauth_secret=oauth_secret,
+                                        method='POST')
+        getattr(TestApp(application), 'post')(
+            signed_uri,
+            params=jsondata,
             content_type='application/json'
         )
 
-        response.getcode = lambda: response.status_int
-        return response
+    from treeherder.client import TreeherderClient
+    monkeypatch.setattr(TreeherderClient, 'post_collection', _post_collection)
 
-    from treeherder.client import TreeherderRequest
-    monkeypatch.setattr(TreeherderRequest, 'send', _send)
+
+@pytest.fixture
+def mock_update_parse_status(monkeypatch, set_oauth_credentials):
+    def _update_parse_status(th_client, project, oauth_key, oauth_secret,
+                             job_log_url_id, parse_status, timestamp=None):
+        if timestamp is None:
+            timestamp = time.time()
+        jsondata = json.dumps({'parse_status': parse_status,
+                               'parse_timestamp': timestamp})
+        signed_uri = th_client._get_uri(
+            project,
+            th_client.UPDATE_ENDPOINT.format(job_log_url_id),
+            data=jsondata, oauth_key=oauth_key, oauth_secret=oauth_secret,
+            method='POST')
+
+        getattr(TestApp(application), 'post')(
+            signed_uri,
+            params=jsondata,
+            content_type='application/json'
+        )
+
+    from treeherder.client import TreeherderClient
+    monkeypatch.setattr(TreeherderClient, 'update_parse_status', _update_parse_status)
 
 
 @pytest.fixture

tests/e2e/conftest.py

@@ -72,7 +72,8 @@ def running_jobs_stored(
 
 @pytest.fixture
 def completed_jobs_stored(
-        jm, completed_jobs, result_set_stored, mock_send_request):
+        jm, completed_jobs, result_set_stored, mock_post_collection,
+        mock_update_parse_status):
     """
     stores a list of buildapi completed jobs into the objectstore
     """

tests/e2e/test_client_job_ingestion.py

@@ -9,7 +9,7 @@ from treeherder.model.derived import JobsModel
 
 
 def test_post_job_with_parsed_log(test_project, result_set_stored,
-                                  mock_send_request):
+                                  mock_post_collection):
     """
     test submitting a job with a pre-parsed log gets the right job_log_url
     parse_status value.
@@ -34,18 +34,13 @@ def test_post_job_with_parsed_log(test_project, result_set_stored,
 
     tjc.add(tj)
 
-    req = client.TreeherderRequest(
-        protocol='http',
-        host='localhost',
-        project=test_project,
-        oauth_key=credentials['consumer_key'],
-        oauth_secret=credentials['consumer_secret']
-        )
+    cli = client.TreeherderClient(protocol='http', host='localhost')
 
     # Post the request to treeherder
-    resp = req.post(tjc)
-    assert resp.status_int == 200
-    assert resp.body == '{"message": "well-formed JSON stored"}'
+    cli.post_collection(test_project, credentials['consumer_key'],
+                        credentials['consumer_secret'], tjc)
+
+    # assume if there were no exceptions we're ok
 
     with JobsModel(test_project) as jm:
         jm.process_objects(10)

tests/etl/conftest.py

@@ -9,7 +9,7 @@ from treeherder.etl.mixins import OAuthLoaderMixin
 from treeherder.etl.oauth_utils import OAuthCredentials
 from treeherder.webapp.wsgi import application
 
-from treeherder.client import TreeherderRequest
+from treeherder.client import TreeherderClient
 from tests.sampledata import SampleData
 
 
@@ -23,18 +23,14 @@ def mock_post_json_data(monkeypatch, jm):
             OAuthCredentials.set_credentials(SampleData.get_credentials())
             credentials = OAuthCredentials.get_credentials(jm.project)
 
-            tr = TreeherderRequest(
-                protocol='http',
-                host='localhost',
-                project=jm.project,
-                oauth_key=credentials['consumer_key'],
-                oauth_secret=credentials['consumer_secret']
-            )
-            signed_uri = tr.oauth_client.get_signed_uri(
-                th_collection.to_json(),
-                tr.get_uri(th_collection.endpoint_base),
-                "POST"
-            )
+            cli = TreeherderClient(protocol='http',
+                                   host='localhost')
+
+            signed_uri = cli._get_uri(jm.project, th_collection.endpoint_base,
+                                      data=th_collection.to_json(),
+                                      oauth_key=credentials['consumer_key'],
+                                      oauth_secret=credentials['consumer_secret'],
+                                      method='POST')
 
             response = TestApp(application).post_json(
                 str(signed_uri), params=th_collection.get_collection_data()

tests/log_parser/test_tasks.py

@@ -76,7 +76,8 @@ def mock_mozlog_get_log_handler(monkeypatch):
 
 
 def test_parse_log(jm, initial_data, jobs_with_local_log, sample_resultset,
-                   mock_send_request, mock_get_remote_content):
+                   mock_post_collection, mock_update_parse_status,
+                   mock_get_remote_content):
     """
     check that at least 3 job_artifacts get inserted when running
     a parse_log task for a successful job
@@ -115,7 +116,8 @@ def test_parse_log(jm, initial_data, jobs_with_local_log, sample_resultset,
 # json-log parsing is disabled due to bug 1152681.
 @pytest.mark.xfail
 def test_parse_mozlog_log(jm, initial_data, jobs_with_local_mozlog_log,
-                          sample_resultset, mock_send_request,
+                          sample_resultset, mock_post_collection,
+                          mock_update_parse_status,
                           mock_get_remote_content,
                           mock_mozlog_get_log_handler
                           ):
@@ -159,7 +161,8 @@ def test_parse_mozlog_log(jm, initial_data, jobs_with_local_mozlog_log,
 
 
 def test_parse_talos_log(jm, test_project, initial_data, jobs_with_local_talos_log,
-                         sample_resultset, mock_send_request, mock_get_remote_content):
+                         sample_resultset, mock_post_collection, mock_update_parse_status,
+                         mock_get_remote_content):
     """
     check that performance job_artifacts get inserted when running
     a parse_log task for a talos job
@@ -177,7 +180,8 @@ def test_parse_talos_log(jm, test_project, initial_data, jobs_with_local_talos_l
 
 
 def test_bug_suggestions_artifact(jm, initial_data, jobs_with_local_log,
-                                  sample_resultset, mock_send_request,
+                                  sample_resultset, mock_post_collection,
+                                  mock_update_parse_status,
                                   mock_get_remote_content
                                   ):
     """

tests/test_utils.py

@@ -10,7 +10,7 @@ from treeherder.model.derived.refdata import RefDataManager
 from treeherder.etl.oauth_utils import OAuthCredentials
 from treeherder.webapp.wsgi import application
 
-from treeherder.client import TreeherderRequest
+from treeherder.client import TreeherderClient
 from tests.sampledata import SampleData
 
 
@@ -31,55 +31,21 @@ def post_collection(
     if consumer_secret:
         credentials['consumer_secret'] = consumer_secret
 
-    tr = TreeherderRequest(
+    cli = TreeherderClient(
         protocol='http',
         host='localhost',
-        project=project,
-        oauth_key=credentials['consumer_key'],
-        oauth_secret=credentials['consumer_secret']
     )
 
-    signed_uri = tr.oauth_client.get_signed_uri(
-        th_collection.to_json(),
-        tr.get_uri(th_collection.endpoint_base),
-        'POST'
-    )
+    jsondata = th_collection.to_json()
+    signed_uri = cli._get_uri(project, th_collection.endpoint_base,
+                              data=jsondata,
+                              oauth_key=credentials['consumer_key'],
+                              oauth_secret=credentials['consumer_secret'],
+                              method='POST')
 
     response = TestApp(application).post_json(
-        str(signed_uri), params=th_collection.get_collection_data(), status=status
-    )
-
-    return response
-
-
-def post_job_data(
-        project, uri, data, status=None, expect_errors=False):
-
-    # Since the uri is passed in it's not generated by the
-    # treeherder request or collection and is missing the protocol
-    # and host. Add those missing elements here.
-    uri = 'http://localhost{0}'.format(uri)
-
-    # Set the credentials
-    OAuthCredentials.set_credentials(SampleData.get_credentials())
-
-    credentials = OAuthCredentials.get_credentials(project)
-
-    tr = TreeherderRequest(
-        protocol='http',
-        host='localhost',
-        project=project,
-        oauth_key=credentials['consumer_key'],
-        oauth_secret=credentials['consumer_secret']
-    )
-
-    signed_uri = tr.get_signed_uri(
-        json.dumps(data), uri
-    )
-
-    response = TestApp(application).post_json(
-        str(signed_uri), params=data, status=status,
-        expect_errors=expect_errors
+        str(signed_uri), params=th_collection.get_collection_data(),
+        status=status
     )
 
     return response

treeherder/client/thclient/client.py

@@ -645,32 +645,24 @@ class OauthClient(object):
         return req.to_url()
 
 
-class TreeherderRequest(object):
+class TreeherderClient(object):
     """
-    Treeherder request object that manages test submission.
+    Treeherder client class
     """
 
     PROTOCOLS = {'http', 'https'}  # supported protocols
-    HEADERS = {'Content-Type': 'application/json'}
+
+    UPDATE_ENDPOINT = 'job-log-url/{}/update_parse_status'
 
     def __init__(
-            self, protocol='', host='', project='', oauth_key='',
-            oauth_secret='', timeout=120):
+            self, protocol='https', host='treeherder.mozilla.org',
+            timeout=120):
         """
-        - host : treeherder host to post to
-        - project : name of the project in treeherder
-        - oauth_key, oauth_secret : oauth credentials
-        - timeout : maximum timeout for requests
+        :param protocol: protocol to use (http or https)
+        :param host: treeherder host to post to
+        :param timeout: maximum time it can take for a request to complete
         """
         self.host = host
-        self.project = project
-        self.oauth_key = oauth_key
-        self.oauth_secret = oauth_secret
-        self.use_oauth = bool(self.oauth_key and self.oauth_secret)
-        self.oauth_client = None
-        if self.use_oauth:
-            self.oauth_client = OauthClient(oauth_key, oauth_secret,
-                                            self.project)
 
         if protocol not in self.PROTOCOLS:
             raise AssertionError('Protocol "%s" not supported; please use one '
@@ -679,17 +671,46 @@ class TreeherderRequest(object):
         self.protocol = protocol
         self.timeout = timeout
 
-        # ensure the required parameters are given
-        if not self.project:
-            msg = "{0}: project required for posting".format(
-                self.__class__.__name__)
-            raise TreeherderClientError(msg, [])
+    def _get_uri(self, project, endpoint, data=None, oauth_key=None,
+                 oauth_secret=None, method='GET'):
 
-    def post(self, collection_inst):
-        """Shortcut method to send a treeherder collection via POST
+        uri = '{0}://{1}/api/project/{2}/{3}/'.format(
+            self.protocol, self.host, project, endpoint
+            )
 
+        if oauth_key and oauth_secret:
+            oauth_client = OauthClient(oauth_key, oauth_secret, project)
+            uri = oauth_client.get_signed_uri(data, uri, method)
+
+        return uri
+
+    def _post_json(self, project, endpoint, oauth_key, oauth_secret, jsondata,
+                   timeout):
+        if timeout is None:
+            timeout = self.timeout
+
+        if not oauth_key or not oauth_secret:
+            raise TreeherderClientError("Must provide oauth key and secret "
+                                        "to post to treeherder!", [])
+
+        uri = self._get_uri(project, endpoint, data=jsondata, oauth_key=oauth_key,
+                            oauth_secret=oauth_secret, method='POST')
+
+        resp = requests.post(uri, data=jsondata,
+                             headers={'Content-Type': 'application/json'},
+                             timeout=timeout)
+        resp.raise_for_status()
+
+    def post_collection(self, project, oauth_key, oauth_secret,
+                        collection_inst, timeout=None):
+        """
+        Sends a treeherder collection to the server
+
+        :param project: project to submit data for
+        :param oauth_key: oauth key credential
+        :param oauth_secret: oauth secret credential
         :param collection_inst: a TreeherderCollection instance
-        :returns: a requests Response object
+        :param timeout: custom timeout in seconds (defaults to class timeout)
         """
 
         if not isinstance(collection_inst, TreeherderCollection):
@@ -715,66 +736,33 @@ class TreeherderRequest(object):
 
         collection_inst.validate()
 
-        return self.send(collection_inst.endpoint_base,
-                         method="POST",
-                         data=collection_inst.to_json())
+        self._post_json(project, collection_inst.endpoint_base, oauth_key,
+                        oauth_secret, collection_inst.to_json(),
+                        timeout=timeout)
 
-    def send(self, endpoint, method=None, data=None):
-        """send data to the given endpoint with the given http method.
-
-        :param endpoint: the target endpoint for this request
-        :param method: can be one of GET,POST,PUT
-        :param data: the body of this request
-        :returns: a requests Response object
+    def update_parse_status(self, project, oauth_key, oauth_secret,
+                            job_log_url_id, parse_status, timestamp=None,
+                            timeout=None):
         """
+        Updates the parsing status of a treeherder job
 
-        req_method = self._get_requests_method(method)
+        :param project: project to submit data for
+        :param oauth_key: oauth key credential
+        :param oauth_secret: oauth secret credential
+        :param parse_status: string representing parse status of a treeherder
+                             job
+        :param timestamp: timestamp of when parse status was updated (defaults
+                          to now)
+        :param timeout: custom timeout in seconds (defaults to class timeout)
+        """
+        if timestamp is None:
+            timestamp = time.time()
 
-        if data:
-            if not isinstance(data, str):
-                # if the body is not serialized yet, do it now
-                serialized_body = json.dumps(data)
-            else:
-                serialized_body = data
-        else:
-            serialized_body = None
-
-        uri = self.get_uri(endpoint)
-
-        if self.use_oauth:
-            uri = self.oauth_client.get_signed_uri(serialized_body, uri, method)
-
-        # Make the request
-        response = req_method(uri,
-                              data=serialized_body,
-                              headers=self.HEADERS,
-                              timeout=self.timeout
-                              )
-
-        return response
-
-    def _get_requests_method(self, method):
-        try:
-            methods = {
-                "GET": requests.get,
-                "POST": requests.post,
-                "PUT": requests.put
-            }
-            return methods[method]
-        except KeyError:
-            msg = "{0}: {1} is not a supported method".format(
-                self.__class__.__name__,
-                method
-            )
-            raise TreeherderClientError(msg, [])
-
-    def get_uri(self, endpoint):
-
-        uri = '{0}://{1}/api/project/{2}/{3}/'.format(
-            self.protocol, self.host, self.project, endpoint
-            )
-
-        return uri
+        self._post_json(project, self.UPDATE_ENDPOINT.format(job_log_url_id),
+                        oauth_key, oauth_secret,
+                        json.dumps({'parse_status': parse_status,
+                                    'parse_timestamp': timestamp}),
+                        timeout=timeout)
 
 
 class TreeherderClientError(Exception):

treeherder/etl/th_publisher.py

@@ -7,7 +7,7 @@ import logging
 from django.utils.encoding import python_2_unicode_compatible
 from django.conf import settings
 
-from treeherder.client import TreeherderRequest
+from treeherder.client import TreeherderClient
 
 from treeherder.etl.oauth_utils import OAuthCredentials
 
@@ -20,24 +20,24 @@ def post_treeherder_collections(th_collections):
 
         credentials = OAuthCredentials.get_credentials(project)
 
-        th_request = TreeherderRequest(
+        cli = TreeherderClient(
             protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
             host=settings.TREEHERDER_REQUEST_HOST,
-            project=project,
-            oauth_key=credentials.get('consumer_key', None),
-            oauth_secret=credentials.get('consumer_secret', None)
         )
 
         logger.info(
-            "collection loading request: {0}".format(
-                th_request.get_uri(th_collections[project].endpoint_base)))
-        response = th_request.post(th_collections[project])
-
-        if not response or response.status_code != 200:
+            "collection loading request for project {0}: {1}".format(
+                project,
+                th_collections[project].endpoint_base))
+        try:
+            cli.post_collection(project, credentials.get('consumer_key'),
+                                credentials.get('consumer_secret'),
+                                th_collections[project])
+        except Exception, e:
             errors.append({
                 "project": project,
                 "url": th_collections[project].endpoint_base,
-                "message": response.text
+                "message": str(e)
             })
 
     if errors:

treeherder/log_parser/utils.py

@@ -4,7 +4,6 @@
 
 import urllib2
 import logging
-import time
 
 import simplejson as json
 from django.conf import settings
@@ -14,8 +13,7 @@ from treeherder.log_parser.artifactbuildercollection import \
 from treeherder.log_parser.artifactbuilders import MozlogArtifactBuilder
 
 from treeherder.model.error_summary import get_error_summary_artifacts
-
-from treeherder.client import TreeherderArtifactCollection, TreeherderRequest
+from treeherder.client import TreeherderClient, TreeherderArtifactCollection
 from treeherder.etl.oauth_utils import OAuthCredentials
 
 logger = logging.getLogger(__name__)
@@ -82,19 +80,18 @@ def post_log_artifacts(project,
     logger.debug("Downloading/parsing log for %s", log_description)
 
     credentials = OAuthCredentials.get_credentials(project)
-    req = TreeherderRequest(
+    client = TreeherderClient(
         protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
-        host=settings.TREEHERDER_REQUEST_HOST,
-        project=project,
-        oauth_key=credentials.get('consumer_key', None),
-        oauth_secret=credentials.get('consumer_secret', None),
+        host=settings.TREEHERDER_REQUEST_HOST
     )
 
     try:
         artifact_list = extract_artifacts_cb(job_log_url['url'],
                                              job_guid, check_errors)
     except Exception as e:
-        update_parse_status(req, job_log_url, 'failed')
+        client.update_parse_status(project, credentials.get('consumer_key'),
+                                   credentials.get('consumer_secret'),
+                                   job_log_url['id'], 'failed')
         if isinstance(e, urllib2.HTTPError) and e.code in (403, 404):
             logger.debug("Unable to retrieve log for %s: %s", log_description, e)
             return
@@ -108,22 +105,13 @@ def post_log_artifacts(project,
         tac.add(ta)
 
     try:
-        req.post(tac)
-        update_parse_status(req, job_log_url, 'parsed')
+        client.post_collection(project, credentials.get('consumer_key'),
+                               credentials.get('consumer_secret'),
+                               tac)
+        client.update_parse_status(project, credentials.get('consumer_key'),
+                                   credentials.get('consumer_secret'),
+                                   job_log_url['id'], 'parsed')
         logger.debug("Finished posting artifact for %s %s", project, job_guid)
     except Exception as e:
         logger.error("Failed to upload parsed artifact for %s: %s", log_description, e)
         _retry(e)
-
-
-def update_parse_status(req, job_log_url, parse_status):
-    update_endpoint = 'job-log-url/{}/update_parse_status'.format(job_log_url['id'])
-    current_timestamp = time.time()
-    req.send(
-        update_endpoint,
-        method='POST',
-        data={
-            'parse_status': parse_status,
-            'parse_timestamp': current_timestamp
-        }
-    )