Bug 1209555 - switch etl requests to hawk

I added a create_credentials command to help setting up the initial
development environment. The puppet setup now creates a new user and set
it as the owner of the treeherder-etl credentials.

puppet/manifests/classes/dev.pp

@@ -19,10 +19,19 @@ class dev{
     user => "${APP_USER}",
   }
 
-  exec{"export_project_credentials":
+  exec{"create_superuser":
     cwd => "${PROJ_DIR}",
-    command => "bash -c 'source /etc/profile.d/treeherder.sh; ${VENV_DIR}/bin/python manage.py export_project_credentials'",
-    require => Exec["init_datasources"],
+    command => "bash -c 'source /etc/profile.d/treeherder.sh; ${VENV_DIR}/bin/python manage.py createsuperuser --username treeherder --email treeherder@mozilla.com --noinput'",
+    require => Exec["init_master_db"],
     user => "${APP_USER}",
+    unless => "bash -c 'source /etc/profile.d/treeherder.sh; ${VENV_DIR}/bin/python manage.py dumpdata auth.User | grep treeherder@mozilla.com'",
+  }
+
+  exec{"create_etl_credentials":
+    cwd => "${PROJ_DIR}",
+    command => "bash -c 'source /etc/profile.d/treeherder.sh; ${VENV_DIR}/bin/python manage.py create_credentials treeherder-etl treeherder@mozilla.com \"Treeherder etl service credentials\"'",
+    require => Exec["create_superuser"],
+    user => "${APP_USER}",
+    unless => "bash -c 'source /etc/profile.d/treeherder.sh; ${VENV_DIR}/bin/python manage.py dumpdata credentials.Credentials | grep treeherder-etl'",
   }
 }

tests/conftest.py

@@ -8,9 +8,11 @@ import pytest
 import responses
 from django.core.management import call_command
 from requests import Request
+from requests_hawk import HawkAuth
 from webtest.app import TestApp
 
 from tests.sampledata import SampleData
+from treeherder.client import TreeherderClient
 from treeherder.etl.oauth_utils import OAuthCredentials
 from treeherder.webapp.wsgi import application
 
@@ -338,23 +340,31 @@ def set_oauth_credentials():
 
 
 @pytest.fixture
-def mock_post_json(monkeypatch, set_oauth_credentials):
+def mock_post_json(monkeypatch, client_credentials):
     def _post_json(th_client, project, endpoint, data,
                    timeout=None, auth=None):
 
+        auth = auth or th_client.auth
+        if not auth:
+            auth = HawkAuth(credentials={
+                'id': client_credentials.client_id,
+                'key': str(client_credentials.secret),
+                'algorithm': 'sha256'
+            })
+        app = TestApp(application)
         uri = th_client._get_project_uri(project, endpoint)
-
         req = Request('POST', uri, json=data, auth=auth)
-
         prepped_request = req.prepare()
 
-        getattr(TestApp(application), 'post')(
+        getattr(app, 'post')(
             prepped_request.url,
             params=json.dumps(data),
-            content_type='application/json'
+            content_type='application/json',
+            extra_environ={
+                'HTTP_AUTHORIZATION': str(prepped_request.headers['Authorization'])
+            }
         )
 
-    from treeherder.client import TreeherderClient
     monkeypatch.setattr(TreeherderClient, '_post_json', _post_json)
 
 
@@ -501,3 +511,34 @@ def retriggers(jm, eleven_jobs_stored):
                placeholders=[retrigger['job_guid'], original['job_guid']])
 
     return [retrigger]
+
+
+@pytest.fixture
+def api_user(request):
+    from django.contrib.auth.models import User
+    user = User.objects.create_user('MyUser')
+
+    def fin():
+        user.delete()
+    request.addfinalizer(fin)
+
+    return user
+
+
+@pytest.fixture
+def client_credentials(request, api_user):
+    from django.conf import settings
+    from treeherder.credentials.models import Credentials
+
+    # We need to get_or_create here because of bug 1133273.
+    # It can be a straight create once that bug is solved.
+    client_credentials, _ = Credentials.objects.get_or_create(
+        client_id=settings.ETL_CLIENT_ID,
+        defaults={'owner': api_user, 'authorized': True}
+    )
+
+    def fin():
+        client_credentials.delete()
+    request.addfinalizer(fin)
+
+    return client_credentials

tests/e2e/conftest.py

@@ -37,13 +37,14 @@ def completed_jobs(sample_data):
 
 @pytest.fixture
 def pending_jobs_stored(
-        jm, pending_jobs, result_set_stored):
+        jm, pending_jobs, result_set_stored, mock_post_json):
     """
     stores a list of buildapi pending jobs into the jobs store
     using BuildApiTreeHerderAdapter
     """
 
     pending_jobs.update(result_set_stored[0])
+    pending_jobs.update({'project': jm.project})
 
     tjc = TreeherderJobCollection()
     tj = tjc.get_job(pending_jobs)
@@ -54,11 +55,12 @@ def pending_jobs_stored(
 
 @pytest.fixture
 def running_jobs_stored(
-        jm, running_jobs, result_set_stored):
+        jm, running_jobs, result_set_stored, mock_post_json):
     """
     stores a list of buildapi running jobs
     """
     running_jobs.update(result_set_stored[0])
+    running_jobs.update({'project': jm.project})
 
     tjc = TreeherderJobCollection()
     tj = tjc.get_job(running_jobs)
@@ -74,6 +76,7 @@ def completed_jobs_stored(
     stores a list of buildapi completed jobs
     """
     completed_jobs['revision_hash'] = result_set_stored[0]['revision_hash']
+    completed_jobs.update({'project': jm.project})
 
     tjc = TreeherderJobCollection()
     tj = tjc.get_job(completed_jobs)

tests/e2e/test_client_job_ingestion.py

@@ -3,9 +3,8 @@ import json
 import pytest
 from mock import MagicMock
 
-from treeherder.client.thclient import (TreeherderAuth,
-                                        client)
-from treeherder.etl.oauth_utils import OAuthCredentials
+from tests.test_utils import post_collection
+from treeherder.client.thclient import client
 from treeherder.log_parser.parsers import StepParser
 from treeherder.model import error_summary
 from treeherder.model.derived import (ArtifactsModel,
@@ -31,17 +30,6 @@ def text_log_summary_dict():
     }
 
 
-def do_post_collection(project, collection):
-    # assume if there were no exceptions we're ok
-    cli = client.TreeherderClient(protocol='http', host='localhost')
-    credentials = OAuthCredentials.get_credentials(project)
-    auth = TreeherderAuth(credentials['consumer_key'],
-                          credentials['consumer_secret'],
-                          project)
-
-    cli.post_collection(project, collection, auth=auth)
-
-
 def check_artifacts(test_project,
                     job_guid,
                     parse_status,
@@ -105,7 +93,7 @@ def test_post_job_with_parsed_log(test_project, result_set_stored,
     })
     tjc.add(tj)
 
-    do_post_collection(test_project, tjc)
+    post_collection(test_project, tjc)
 
     check_artifacts(test_project, job_guid, 'parsed', 0)
 
@@ -153,7 +141,7 @@ def test_post_job_with_text_log_summary_artifact_parsed(
     })
     tjc.add(tj)
 
-    do_post_collection(test_project, tjc)
+    post_collection(test_project, tjc)
 
     check_artifacts(test_project, job_guid, 'parsed', 2,
                     {'Bug suggestions', 'text_log_summary'}, mock_error_summary)
@@ -202,7 +190,7 @@ def test_post_job_with_text_log_summary_artifact_parsed_dict_blob(
     })
     tjc.add(tj)
 
-    do_post_collection(test_project, tjc)
+    post_collection(test_project, tjc)
 
     check_artifacts(test_project, job_guid, 'parsed', 2,
                     {'Bug suggestions', 'text_log_summary'}, mock_error_summary)
@@ -253,7 +241,7 @@ def test_post_job_with_text_log_summary_artifact_pending(
 
     tjc.add(tj)
 
-    do_post_collection(test_project, tjc)
+    post_collection(test_project, tjc)
 
     check_artifacts(test_project, job_guid, 'parsed', 2,
                     {'Bug suggestions', 'text_log_summary'}, mock_error_summary)
@@ -315,7 +303,7 @@ def test_post_job_with_text_log_summary_and_bug_suggestions_artifact(
 
     tjc.add(tj)
 
-    do_post_collection(test_project, tjc)
+    post_collection(test_project, tjc)
 
     check_artifacts(test_project, job_guid, 'parsed', 2,
                     {'Bug suggestions', 'text_log_summary'}, error_summary_blob)
@@ -384,7 +372,7 @@ def test_post_job_artifacts_by_add_artifact(
 
     tjc.add(tj)
 
-    do_post_collection(test_project, tjc)
+    post_collection(test_project, tjc)
 
     check_artifacts(test_project, job_guid, 'parsed', 5,
                     {'Bug suggestions', 'text_log_summary', 'Job Info',
@@ -411,7 +399,7 @@ def test_post_job_with_tier(test_project, result_set_stored,
     tj.add_tier(3)
     tjc.add(tj)
 
-    do_post_collection(test_project, tjc)
+    post_collection(test_project, tjc)
 
     with JobsModel(test_project) as jobs_model:
         job = [x for x in jobs_model.get_job_list(0, 20)
@@ -435,7 +423,7 @@ def test_post_job_with_default_tier(test_project, result_set_stored,
     })
     tjc.add(tj)
 
-    do_post_collection(test_project, tjc)
+    post_collection(test_project, tjc)
 
     with JobsModel(test_project) as jobs_model:
         job = [x for x in jobs_model.get_job_list(0, 20)

tests/e2e/test_perf_ingestion.py

@@ -1,5 +1,5 @@
-from test_client_job_ingestion import do_post_collection
 from tests.sampledata import SampleData
+from tests.test_utils import post_collection
 from treeherder.client.thclient import client
 from treeherder.perf.models import (PerformanceDatum,
                                     PerformanceSignature)
@@ -36,7 +36,7 @@ def test_post_talos_artifact(test_project, test_repository, result_set_stored,
 
     tjc.add(tj)
 
-    do_post_collection(test_project, tjc)
+    post_collection(test_project, tjc)
 
     # we'll just validate that we got the expected number of results for
     # talos (we have validation elsewhere for the actual data adapters)

tests/test_utils.py

@@ -1,46 +1,14 @@
 import json
 
-from requests import Request
-from webtest.app import TestApp
-
 from tests.sampledata import SampleData
-from treeherder.client import (TreeherderAuth,
-                               TreeherderClient)
-from treeherder.etl.oauth_utils import OAuthCredentials
+from treeherder.client import TreeherderClient
 from treeherder.model.derived.refdata import RefDataManager
-from treeherder.webapp.wsgi import application
 
 
-def post_collection(
-        project, th_collection, status=None, expect_errors=False,
-        consumer_key=None, consumer_secret=None):
+def post_collection(project, th_collection, auth=None):
 
-    # Set the credentials
-    OAuthCredentials.set_credentials(SampleData.get_credentials())
-
-    credentials = OAuthCredentials.get_credentials(project)
-
-    # The only time the credentials should be overridden are when
-    # a client needs to test authentication failure confirmation
-    consumer_key = consumer_key or credentials['consumer_key']
-    consumer_secret = consumer_secret or credentials['consumer_secret']
-
-    auth = TreeherderAuth(consumer_key, consumer_secret, project)
-    client = TreeherderClient(protocol='http', host='localhost', auth=auth)
-    uri = client._get_project_uri(project, th_collection.endpoint_base)
-
-    req = Request('POST', uri,
-                  json=th_collection.get_collection_data(),
-                  auth=auth)
-    prepped_request = req.prepare()
-
-    response = TestApp(application).post_json(
-        prepped_request.url,
-        params=th_collection.get_collection_data(),
-        status=status
-    )
-
-    return response
+    client = TreeherderClient(protocol='http', host='localhost', auth=None)
+    return client.post_collection(project, th_collection)
 
 
 def do_job_ingestion(jm, refdata, job_data, sample_resultset, verify_data=True):

tests/webapp/api/test_artifact_api.py

@@ -3,9 +3,7 @@ import json
 import pytest
 from django.core.urlresolvers import reverse
 
-from treeherder.client.thclient import (TreeherderAuth,
-                                        client)
-from treeherder.etl.oauth_utils import OAuthCredentials
+from treeherder.client.thclient import client
 from treeherder.model.derived import (ArtifactsModel,
                                       JobsModel)
 
@@ -92,11 +90,7 @@ def test_artifact_create_text_log_summary(webapp, test_project, eleven_jobs_stor
     })
     tac.add(ta)
 
-    credentials = OAuthCredentials.get_credentials(test_project)
-    auth = TreeherderAuth(credentials['consumer_key'],
-                          credentials['consumer_secret'],
-                          test_project)
-    cli = client.TreeherderClient(protocol='http', host='localhost', auth=auth)
+    cli = client.TreeherderClient(protocol='http', host='localhost')
     cli.post_collection(test_project,  tac)
 
     with ArtifactsModel(test_project) as artifacts_model:
@@ -141,11 +135,7 @@ def test_artifact_create_text_log_summary_and_bug_suggestions(
         'job_guid': job['job_guid']
     }))
 
-    credentials = OAuthCredentials.get_credentials(test_project)
-    auth = TreeherderAuth(credentials['consumer_key'],
-                          credentials['consumer_secret'],
-                          test_project)
-    cli = client.TreeherderClient(protocol='http', host='localhost', auth=auth)
+    cli = client.TreeherderClient(protocol='http', host='localhost')
     cli.post_collection(test_project, tac)
 
     with ArtifactsModel(test_project) as artifacts_model:

tests/webapp/api/test_auth.py

@@ -1,6 +1,4 @@
 import oauth2 as oauth
-import pytest
-from django.contrib.auth.models import User
 from django.core.urlresolvers import (resolve,
                                       reverse)
 from mohawk import Sender
@@ -8,7 +6,6 @@ from rest_framework.decorators import APIView
 from rest_framework.response import Response
 from rest_framework.test import APIRequestFactory
 
-from treeherder.credentials.models import Credentials
 from treeherder.etl.oauth_utils import OAuthCredentials
 from treeherder.webapp.api import permissions
 
@@ -98,29 +95,6 @@ def test_two_legged_oauth_project_via_user(monkeypatch, jm, set_oauth_credential
     assert response.data == {'authenticated': True}
 
 
-@pytest.fixture
-def api_user(request):
-    user = User.objects.create_user('MyUser')
-
-    def fin():
-        user.delete()
-    request.addfinalizer(fin)
-
-    return user
-
-
-@pytest.fixture
-def client_credentials(request, api_user):
-    client_credentials = Credentials.objects.create(
-        client_id='test-credentials', owner=api_user)
-
-    def fin():
-        client_credentials.delete()
-    request.addfinalizer(fin)
-
-    return client_credentials
-
-
 def _get_hawk_response(client_id, secret, method='GET',
                        content='', content_type='application/json'):
     auth = {
@@ -158,6 +132,8 @@ def test_get_hawk_authorized(client_credentials):
 
 
 def test_get_hawk_unauthorized(client_credentials):
+    client_credentials.authorized = False
+    client_credentials.save()
     response = _get_hawk_response(client_credentials.client_id,
                                   str(client_credentials.secret))
     assert response.data == {'detail': ('No authentication credentials '
@@ -174,6 +150,8 @@ def test_post_hawk_authorized(client_credentials):
 
 
 def test_post_hawk_unauthorized(client_credentials):
+    client_credentials.authorized = False
+    client_credentials.save()
     response = _get_hawk_response(client_credentials.client_id,
                                   str(client_credentials.secret), method='POST',
                                   content="{'this': 'that'}")

tests/webapp/api/test_resultset_api.py

@@ -216,7 +216,7 @@ def test_result_set_detail_bad_project(webapp, jm):
     assert resp.json == {"detail": "No project with name foo"}
 
 
-def test_resultset_create(sample_resultset, jm, initial_data):
+def test_resultset_create(sample_resultset, jm, initial_data, mock_post_json):
     """
     test posting data to the resultset endpoint via webtest.
     extected result are:
@@ -228,13 +228,11 @@ def test_resultset_create(sample_resultset, jm, initial_data):
     trsc = TreeherderResultSetCollection()
 
     for rs in sample_resultset:
-        rs = trsc.get_resultset(rs)
-        trsc.add(rs)
+        rs.update({'author': 'John Doe'})
+        result_set = trsc.get_resultset(rs)
+        trsc.add(result_set)
 
-    resp = test_utils.post_collection(jm.project, trsc)
-
-    assert resp.status_int == 200
-    assert resp.json['message'] == 'well-formed JSON stored'
+    test_utils.post_collection(jm.project, trsc)
 
     stored_objs = jm.get_dhub().execute(
         proc="jobs_test.selects.resultset_by_rev_hash",
@@ -247,36 +245,6 @@ def test_resultset_create(sample_resultset, jm, initial_data):
     jm.disconnect()
 
 
-def test_resultset_with_bad_secret(sample_resultset, jm, initial_data):
-
-    trsc = TreeherderResultSetCollection()
-    for rs in sample_resultset:
-        rs = trsc.get_resultset(rs)
-        trsc.add(rs)
-
-    resp = test_utils.post_collection(
-        jm.project, trsc, status=403, consumer_secret="horrible secret"
-    )
-
-    assert resp.status_int == 403
-    assert resp.json['detail'] == "Client authentication failed for project {0}".format(jm.project)
-
-
-def test_resultset_with_bad_key(sample_resultset, jm, initial_data):
-
-    trsc = TreeherderResultSetCollection()
-    for rs in sample_resultset:
-        rs = trsc.get_resultset(rs)
-        trsc.add(rs)
-
-    resp = test_utils.post_collection(
-        jm.project, trsc, status=403, consumer_key="horrible-key"
-    )
-
-    assert resp.status_int == 403
-    assert resp.json['detail'] == 'oauth_consumer_key does not match credentials for project {0}'.format(jm.project)
-
-
 def test_resultset_cancel_all(jm, resultset_with_three_jobs, pulse_action_consumer):
     """
     Issue cancellation of a resultset with three unfinished jobs.

treeherder/credentials/management/commands/create_credentials.py

@@ -0,0 +1,25 @@
+from django.contrib.auth.models import User
+from django.core.management.base import BaseCommand
+
+from treeherder.credentials.models import Credentials
+
+
+class Command(BaseCommand):
+    help = 'Create a new set of credentials to use with the hawk authentication scheme'
+
+    def add_arguments(self, parser):
+        parser.add_argument('client_id', type=str)
+        parser.add_argument('owner', type=str)
+        parser.add_argument('description', type=str)
+
+    def handle(self, *args, **options):
+        owner = User.objects.get(email=options['owner'])
+        credentials = Credentials.objects.create(
+            client_id=options['client_id'],
+            description=options['description'],
+            owner=owner,
+            authorized=True
+        )
+
+        self.stdout.write('Successfully created credentials for "%s"' % credentials.client_id)
+        self.stdout.write('Secret: %s' % credentials.secret)

treeherder/etl/th_publisher.py

@@ -3,10 +3,10 @@ import traceback
 
 from django.conf import settings
 from django.utils.encoding import python_2_unicode_compatible
+from requests_hawk import HawkAuth
 
-from treeherder.client import (TreeherderAuth,
-                               TreeherderClient)
-from treeherder.etl.oauth_utils import OAuthCredentials
+from treeherder.client import TreeherderClient
+from treeherder.credentials.models import Credentials
 
 logger = logging.getLogger(__name__)
 
@@ -14,19 +14,20 @@ logger = logging.getLogger(__name__)
 def post_treeherder_collections(th_collections, chunk_size=1):
 
     errors = []
+    credentials = Credentials.objects.get(client_id=settings.ETL_CLIENT_ID)
+    auth = HawkAuth(credentials={
+        'id': credentials.client_id,
+        'key': str(credentials.secret),
+        'algorithm': 'sha256'
+    })
+
     cli = TreeherderClient(
         protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
         host=settings.TREEHERDER_REQUEST_HOST,
+        auth=auth
     )
 
     for project in th_collections:
-
-        credentials = OAuthCredentials.get_credentials(project)
-
-        auth = TreeherderAuth(credentials.get('consumer_key'),
-                              credentials.get('consumer_secret'),
-                              project)
-
         logger.info(
             "collection loading request for project {0}: {1}".format(
                 project,
@@ -36,7 +37,7 @@ def post_treeherder_collections(th_collections, chunk_size=1):
 
         for collection in collection_chunks:
             try:
-                cli.post_collection(project, collection, auth=auth)
+                cli.post_collection(project, collection)
             except Exception:
                 errors.append({
                     "project": project,

treeherder/log_parser/utils.py

@@ -3,12 +3,12 @@ import urllib2
 
 import simplejson as json
 from django.conf import settings
+from requests_hawk import HawkAuth
 
 from treeherder import celery_app
 from treeherder.client import (TreeherderArtifactCollection,
-                               TreeherderAuth,
                                TreeherderClient)
-from treeherder.etl.oauth_utils import OAuthCredentials
+from treeherder.credentials.models import Credentials
 from treeherder.log_parser.artifactbuildercollection import ArtifactBuilderCollection
 from treeherder.log_parser.artifactbuilders import MozlogArtifactBuilder
 from treeherder.model.error_summary import get_error_summary_artifacts
@@ -82,10 +82,12 @@ def post_log_artifacts(project,
     log_description = "%s %s (%s)" % (project, job_guid, job_log_url['url'])
     logger.debug("Downloading/parsing log for %s", log_description)
 
-    credentials = OAuthCredentials.get_credentials(project)
-    auth = TreeherderAuth(credentials.get('consumer_key'),
-                          credentials.get('consumer_secret'),
-                          project)
+    credentials = Credentials.objects.get(client_id=settings.ETL_CLIENT_ID)
+    auth = HawkAuth(credentials={
+        'id': credentials.client_id,
+        'key': str(credentials.secret),
+        'algorithm': 'sha256'
+    })
     client = TreeherderClient(
         protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
         host=settings.TREEHERDER_REQUEST_HOST,

treeherder/settings/base.py

@@ -473,3 +473,6 @@ REST_FRAMEWORK_EXTENSIONS = {
 }
 
 HAWK_CREDENTIALS_LOOKUP = 'treeherder.webapp.api.auth.hawk_lookup'
+
+# This is the client id used by the internal data ingestion service.
+ETL_CLIENT_ID = 'treeherder-etl'