зеркало из https://github.com/mozilla/treeherder.git
Merge pull request #173 from mozilla/log-parsing-status
add log parsing status handling
This commit is contained in:
Mauro Doglio
Коммит
c8710cb5d6
|
|
@ -26,6 +26,6 @@ httplib2==0.7.4
|
|||
jsonfield==0.9.20
|
||||
|
||||
git+git://github.com/jeads/datasource@143ac08d11
|
||||
git+git://github.com/mozilla/treeherder-client@cc30664ad9
|
||||
git+git://github.com/mozilla/treeherder-client@be6cb763dc
|
||||
git+git://github.com/Julian/jsonschema@1976689051
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,18 @@
|
|||
import os
|
||||
from os.path import dirname
|
||||
import sys
|
||||
from django.core.management import call_command
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from django.core.management import call_command
|
||||
from webtest.app import TestApp
|
||||
|
||||
from thclient.client import TreeherderRequest
|
||||
|
||||
from tests.sampledata import SampleData
|
||||
from treeherder.etl.oauth_utils import OAuthCredentials
|
||||
from treeherder.webapp.wsgi import application
|
||||
|
||||
|
||||
def pytest_addoption(parser):
|
||||
parser.addoption(
|
||||
|
|
@ -309,3 +319,32 @@ def eleven_jobs_stored(jm, sample_data, sample_resultset):
|
|||
def eleven_jobs_processed(jm, mock_log_parser, eleven_jobs_stored):
|
||||
"""stores and processes list of 11 job samples"""
|
||||
jm.process_objects(11, raise_errors=True)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_send_request(monkeypatch, jm):
|
||||
def _send(th_request, endpoint, method=None, data=None):
|
||||
|
||||
OAuthCredentials.set_credentials(SampleData.get_credentials())
|
||||
credentials = OAuthCredentials.get_credentials(jm.project)
|
||||
|
||||
th_request.oauth_key = credentials['consumer_key']
|
||||
th_request.oauth_secret = credentials['consumer_secret']
|
||||
|
||||
if data and not isinstance(data, str):
|
||||
data = json.dumps(data)
|
||||
|
||||
signed_uri = th_request.oauth_client.get_signed_uri(
|
||||
data, th_request.get_uri(endpoint), method
|
||||
)
|
||||
|
||||
response = getattr(TestApp(application), method.lower())(
|
||||
str(signed_uri),
|
||||
params=data,
|
||||
content_type='application/json'
|
||||
)
|
||||
|
||||
response.getcode = lambda: response.status_int
|
||||
return response
|
||||
|
||||
monkeypatch.setattr(TreeherderRequest, 'send', _send)
|
||||
|
|
|
|||
|
|
@ -1,16 +1,10 @@
|
|||
import os
|
||||
|
||||
import pytest
|
||||
import simplejson as json
|
||||
from webtest.app import TestApp
|
||||
|
||||
from django.template import Context, Template
|
||||
from thclient import (TreeherderJobCollection)
|
||||
|
||||
from treeherder.webapp.wsgi import application
|
||||
|
||||
from thclient import (TreeherderJobCollection, TreeherderRequest)
|
||||
|
||||
from treeherder.etl.oauth_utils import OAuthCredentials
|
||||
from tests.sampledata import SampleData
|
||||
from tests import test_utils
|
||||
|
||||
|
||||
|
|
@ -91,27 +85,3 @@ def completed_jobs_loaded(jm, completed_jobs_stored):
|
|||
jm.process_objects(1, raise_errors=True)
|
||||
|
||||
jm.disconnect()
|
||||
|
||||
@pytest.fixture
|
||||
def mock_send_request(monkeypatch, jm):
|
||||
def _send(th_request, th_collection):
|
||||
|
||||
OAuthCredentials.set_credentials(SampleData.get_credentials())
|
||||
credentials = OAuthCredentials.get_credentials(jm.project)
|
||||
|
||||
th_request.oauth_key = credentials['consumer_key']
|
||||
th_request.oauth_secret = credentials['consumer_secret']
|
||||
|
||||
signed_uri = th_request.get_signed_uri(
|
||||
th_collection.to_json(), th_request.get_uri(th_collection)
|
||||
)
|
||||
|
||||
response = TestApp(application).post_json(
|
||||
str(signed_uri), params=th_collection.get_collection_data()
|
||||
)
|
||||
|
||||
|
||||
response.getcode = lambda: response.status_int
|
||||
return response
|
||||
|
||||
monkeypatch.setattr(TreeherderRequest, 'send', _send)
|
||||
|
|
|
|||
|
|
@ -26,8 +26,10 @@ def mock_post_json_data(monkeypatch, jm):
|
|||
oauth_key=credentials['consumer_key'],
|
||||
oauth_secret=credentials['consumer_secret']
|
||||
)
|
||||
signed_uri = tr.get_signed_uri(
|
||||
th_collection.to_json(), tr.get_uri(th_collection)
|
||||
signed_uri = tr.oauth_client.get_signed_uri(
|
||||
th_collection.to_json(),
|
||||
tr.get_uri(th_collection.endpoint_base),
|
||||
"POST"
|
||||
)
|
||||
|
||||
response = TestApp(application).post_json(
|
||||
|
|
|
|||
|
|
@ -1,33 +0,0 @@
|
|||
import pytest
|
||||
from webtest.app import TestApp
|
||||
|
||||
from thclient import TreeherderRequest
|
||||
|
||||
from treeherder.etl.oauth_utils import OAuthCredentials
|
||||
from treeherder.webapp.wsgi import application
|
||||
|
||||
from tests.sampledata import SampleData
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_send_request(monkeypatch, jm):
|
||||
def _send(th_request, th_collection):
|
||||
|
||||
OAuthCredentials.set_credentials(SampleData.get_credentials())
|
||||
credentials = OAuthCredentials.get_credentials(jm.project)
|
||||
|
||||
th_request.oauth_key = credentials['consumer_key']
|
||||
th_request.oauth_secret = credentials['consumer_secret']
|
||||
|
||||
signed_uri = th_request.get_signed_uri(
|
||||
th_collection.to_json(), th_request.get_uri(th_collection)
|
||||
)
|
||||
|
||||
response = TestApp(application).post_json(
|
||||
str(signed_uri), params=th_collection.get_collection_data()
|
||||
)
|
||||
|
||||
response.getcode = lambda: response.status_int
|
||||
return response
|
||||
|
||||
monkeypatch.setattr(TreeherderRequest, 'send', _send)
|
||||
|
|
@ -1,17 +1,15 @@
|
|||
import json
|
||||
import itertools
|
||||
from datadiff import diff
|
||||
from webtest.app import TestApp, AppError
|
||||
|
||||
from sampledata import SampleData
|
||||
from treeherder.model.derived.refdata import RefDataManager
|
||||
from treeherder.etl.mixins import OAuthLoaderMixin
|
||||
from treeherder.etl.oauth_utils import OAuthCredentials
|
||||
from treeherder.webapp.wsgi import application
|
||||
|
||||
from thclient import TreeherderRequest
|
||||
from tests.sampledata import SampleData
|
||||
|
||||
|
||||
def post_collection(
|
||||
project, th_collection, status=None, expect_errors=False,
|
||||
consumer_key=None, consumer_secret=None):
|
||||
|
|
@ -37,16 +35,19 @@ def post_collection(
|
|||
oauth_secret=credentials['consumer_secret']
|
||||
)
|
||||
|
||||
signed_uri = tr.get_signed_uri(
|
||||
th_collection.to_json(), tr.get_uri(th_collection)
|
||||
)
|
||||
signed_uri = tr.oauth_client.get_signed_uri(
|
||||
th_collection.to_json(),
|
||||
tr.get_uri(th_collection.endpoint_base),
|
||||
'POST'
|
||||
)
|
||||
|
||||
response = TestApp(application).post_json(
|
||||
str(signed_uri), params=th_collection.get_collection_data(), status=status
|
||||
)
|
||||
)
|
||||
|
||||
return response
|
||||
|
||||
|
||||
def post_job_data(
|
||||
project, uri, data, status=None, expect_errors=False):
|
||||
|
||||
|
|
|
|||
|
|
@ -129,7 +129,7 @@ class OAuthLoaderMixin(object):
|
|||
oauth_secret=credentials.get('consumer_secret', None)
|
||||
)
|
||||
|
||||
response = th_request.send(th_collections[project])
|
||||
response = th_request.post(th_collections[project])
|
||||
|
||||
if not response or response.status != 200:
|
||||
message = response.read()
|
||||
|
|
|
|||
|
|
@ -59,7 +59,7 @@ def fetch_push_logs():
|
|||
rdm.disconnect()
|
||||
|
||||
|
||||
@task(name='fetch-hg-push-logs', time_limit=30)
|
||||
@task(name='fetch-hg-push-logs', time_limit=60)
|
||||
def fetch_hg_push_log(repo_name, repo_url):
|
||||
"""
|
||||
Run a HgPushlog etl process
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ http://docs.celeryproject.org/en/latest/userguide/canvas.html#guide-canvas
|
|||
"""
|
||||
import simplejson as json
|
||||
import re
|
||||
import urllib
|
||||
import time
|
||||
|
||||
from celery import task
|
||||
from django.conf import settings
|
||||
|
|
@ -16,106 +16,138 @@ from django.core.urlresolvers import reverse
|
|||
|
||||
from thclient import TreeherderArtifactCollection, TreeherderRequest
|
||||
|
||||
from treeherder.log_parser.artifactbuildercollection import ArtifactBuilderCollection
|
||||
from treeherder.log_parser.artifactbuildercollection import \
|
||||
ArtifactBuilderCollection
|
||||
from treeherder.log_parser.utils import (get_error_search_term,
|
||||
get_crash_signature, get_bugs_for_search_term)
|
||||
get_crash_signature,
|
||||
get_bugs_for_search_term)
|
||||
|
||||
from treeherder.etl.oauth_utils import OAuthCredentials
|
||||
|
||||
|
||||
@task(name='parse-log')
|
||||
def parse_log(project, log_url, job_guid, resultset, check_errors=False):
|
||||
def parse_log(project, job_log_url, job_guid, check_errors=False):
|
||||
"""
|
||||
Call ArtifactBuilderCollection on the given job.
|
||||
"""
|
||||
mozharness_pattern = re.compile(
|
||||
'^\d+:\d+:\d+[ ]+(?:DEBUG|INFO|WARNING|ERROR|CRITICAL|FATAL) - [ ]?'
|
||||
)
|
||||
|
||||
bugs_cache = {'open': {}, 'closed': {}}
|
||||
bug_suggestions = {'open': {}, 'closed': {}}
|
||||
|
||||
|
||||
bugscache_uri = '{0}{1}'.format(
|
||||
settings.API_HOSTNAME,
|
||||
reverse("bugscache-list")
|
||||
)
|
||||
|
||||
credentials = OAuthCredentials.get_credentials(project)
|
||||
req = TreeherderRequest(
|
||||
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
|
||||
host=settings.TREEHERDER_REQUEST_HOST,
|
||||
project=project,
|
||||
oauth_key=credentials.get('consumer_key', None),
|
||||
oauth_secret=credentials.get('consumer_secret', None),
|
||||
)
|
||||
update_endpoint = 'job-log-url/{0}/update_parse_status'.format(job_log_url['id'])
|
||||
|
||||
if log_url:
|
||||
# parse a log given its url
|
||||
artifact_bc = ArtifactBuilderCollection(
|
||||
log_url,
|
||||
check_errors=check_errors,
|
||||
|
||||
try:
|
||||
log_url = job_log_url['url']
|
||||
mozharness_pattern = re.compile(
|
||||
'^\d+:\d+:\d+[ ]+(?:DEBUG|INFO|WARNING|ERROR|CRITICAL|FATAL) - [ ]?'
|
||||
)
|
||||
artifact_bc.parse()
|
||||
artifact_list = []
|
||||
for name, artifact in artifact_bc.artifacts.items():
|
||||
if name == 'talos_data':
|
||||
data_type = 'performance'
|
||||
if artifact[name]:
|
||||
artifact_list.append(
|
||||
(job_guid, name, data_type, json.dumps(artifact[name][0]))
|
||||
)
|
||||
else:
|
||||
data_type = 'json'
|
||||
artifact_list.append((job_guid, name, data_type, json.dumps(artifact)))
|
||||
|
||||
if check_errors:
|
||||
all_errors = artifact_bc.artifacts.get(
|
||||
'Structured Log', {}
|
||||
).get(
|
||||
'step_data', {}
|
||||
bugs_cache = {'open': {}, 'closed': {}}
|
||||
bug_suggestions = {'open': {}, 'closed': {}}
|
||||
|
||||
|
||||
# return the resultset with the job id to identify if the UI wants
|
||||
# to fetch the whole thing.
|
||||
|
||||
bugscache_uri = '{0}{1}'.format(
|
||||
settings.API_HOSTNAME,
|
||||
reverse("bugscache-list")
|
||||
)
|
||||
|
||||
if log_url:
|
||||
# parse a log given its url
|
||||
artifact_bc = ArtifactBuilderCollection(
|
||||
log_url,
|
||||
check_errors=check_errors,
|
||||
)
|
||||
artifact_bc.parse()
|
||||
|
||||
artifact_list = []
|
||||
for name, artifact in artifact_bc.artifacts.items():
|
||||
artifact_list.append((job_guid, name, 'json', json.dumps(artifact)))
|
||||
|
||||
if check_errors:
|
||||
all_errors = artifact_bc.artifacts.get(
|
||||
'Structured Log', {}
|
||||
).get(
|
||||
'all_errors', [] )
|
||||
'step_data', {}
|
||||
).get(
|
||||
'all_errors', [] )
|
||||
|
||||
for err in all_errors:
|
||||
# remove the mozharness prefix
|
||||
clean_line = mozharness_pattern.sub('', err['line']).strip()
|
||||
# get a meaningful search term out of the error line
|
||||
search_term = get_error_search_term(clean_line)
|
||||
# collect open and closed bugs suggestions
|
||||
for status in ('open', 'closed'):
|
||||
if not search_term:
|
||||
bug_suggestions[status][clean_line] = []
|
||||
continue
|
||||
if search_term not in bugs_cache[status]:
|
||||
# retrieve the list of suggestions from the api
|
||||
bugs_cache[status][search_term] = get_bugs_for_search_term(
|
||||
search_term,
|
||||
status,
|
||||
bugscache_uri
|
||||
)
|
||||
# no suggestions, try to use the crash signature as search term
|
||||
if not bugs_cache[status][search_term]:
|
||||
crash_signature = get_crash_signature(search_term)
|
||||
if crash_signature:
|
||||
bugs_cache[status][search_term] = get_bugs_for_search_term(
|
||||
search_term,
|
||||
status,
|
||||
bugscache_uri
|
||||
)
|
||||
bug_suggestions[status][clean_line] = bugs_cache[status][search_term]
|
||||
for err in all_errors:
|
||||
# remove the mozharness prefix
|
||||
clean_line = mozharness_pattern.sub('', err['line']).strip()
|
||||
# get a meaningful search term out of the error line
|
||||
search_term = get_error_search_term(clean_line)
|
||||
# collect open and closed bugs suggestions
|
||||
for status in ('open', 'closed'):
|
||||
if not search_term:
|
||||
bug_suggestions[status][clean_line] = []
|
||||
continue
|
||||
if search_term not in bugs_cache[status]:
|
||||
# retrieve the list of suggestions from the api
|
||||
bugs_cache[status][search_term] = get_bugs_for_search_term(
|
||||
search_term,
|
||||
status,
|
||||
bugscache_uri
|
||||
)
|
||||
# no suggestions, try to use the crash signature as search term
|
||||
if not bugs_cache[status][search_term]:
|
||||
crash_signature = get_crash_signature(search_term)
|
||||
if crash_signature:
|
||||
bugs_cache[status][search_term] = get_bugs_for_search_term(
|
||||
search_term,
|
||||
status,
|
||||
bugscache_uri
|
||||
)
|
||||
bug_suggestions[status][clean_line] = bugs_cache[status][search_term]
|
||||
|
||||
artifact_list.append((job_guid, 'Open bugs', 'json', json.dumps(bug_suggestions['open'])))
|
||||
artifact_list.append((job_guid, 'Closed bugs', 'json', json.dumps(bug_suggestions['closed'])))
|
||||
# store the artifacts generated
|
||||
tac = TreeherderArtifactCollection()
|
||||
for artifact in artifact_list:
|
||||
ta = tac.get_artifact({
|
||||
"job_guid": artifact[0],
|
||||
"name": artifact[1],
|
||||
"type": artifact[2],
|
||||
"blob": artifact[3]
|
||||
})
|
||||
tac.add(ta)
|
||||
|
||||
# store the artifacts generated
|
||||
tac = TreeherderArtifactCollection()
|
||||
for artifact in artifact_list:
|
||||
ta = tac.get_artifact({
|
||||
"job_guid": artifact[0],
|
||||
"name": artifact[1],
|
||||
"type": artifact[2],
|
||||
"blob": artifact[3]
|
||||
})
|
||||
tac.add(ta)
|
||||
req = TreeherderRequest(
|
||||
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
|
||||
host=settings.TREEHERDER_REQUEST_HOST,
|
||||
project=project,
|
||||
oauth_key=credentials.get('consumer_key', None),
|
||||
oauth_secret=credentials.get('consumer_secret', None),
|
||||
req.post(tac)
|
||||
|
||||
# send an update to job_log_url
|
||||
# the job_log_url status changes
|
||||
# from pending to running
|
||||
current_timestamp = time.time()
|
||||
status = 'parsed'
|
||||
req.send(
|
||||
update_endpoint,
|
||||
method='POST',
|
||||
data={
|
||||
'parse_status': status,
|
||||
'parse_timestamp': current_timestamp
|
||||
}
|
||||
)
|
||||
|
||||
except Exception, e:
|
||||
# send an update to job_log_url
|
||||
# the job_log_url status changes
|
||||
# from pending to running
|
||||
current_timestamp = time.time()
|
||||
status = 'failed'
|
||||
req.send(
|
||||
update_endpoint,
|
||||
method='POST',
|
||||
data={
|
||||
'parse_status': status,
|
||||
'parse_timestamp': current_timestamp
|
||||
}
|
||||
)
|
||||
req.send(tac)
|
||||
# re raise the exception to leave a trace in the log
|
||||
raise
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ from treeherder.events.publisher import JobStatusPublisher
|
|||
|
||||
from treeherder.etl.common import get_guid_root
|
||||
|
||||
from .base import TreeherderModelBase
|
||||
from .base import TreeherderModelBase, ObjectNotFoundException
|
||||
|
||||
from datasource.DataHub import DataHub
|
||||
|
||||
|
|
@ -1850,9 +1850,6 @@ class JobsModel(TreeherderModelBase):
|
|||
task['routing_key'] = 'parse_log.success'
|
||||
tasks.append(task)
|
||||
|
||||
# a dict of result_set_id => push_timestamp
|
||||
push_timestamp_lookup = self.get_push_timestamp_lookup(result_sets)
|
||||
|
||||
# Store the log references
|
||||
self.get_jobs_dhub().execute(
|
||||
proc='jobs.inserts.set_job_log_url',
|
||||
|
|
@ -1860,18 +1857,67 @@ class JobsModel(TreeherderModelBase):
|
|||
placeholders=log_placeholders,
|
||||
executemany=True)
|
||||
|
||||
# I need to find the jog_log_url ids
|
||||
# just inserted but there's no unique key.
|
||||
# Also, the url column is not indexed, so it's
|
||||
# not a good idea to search based on that.
|
||||
# I'm gonna retrieve the logs by job ids and then
|
||||
# use their url to create a map.
|
||||
|
||||
job_ids = [j["id"] for j in job_id_lookup.values()]
|
||||
|
||||
job_log_url_list = self.get_job_log_url_list(job_ids)
|
||||
|
||||
log_url_lookup = dict([(jlu['url'], jlu)
|
||||
for jlu in job_log_url_list])
|
||||
|
||||
for task in tasks:
|
||||
parse_log.apply_async(
|
||||
args=[
|
||||
self.project,
|
||||
task['log_url'],
|
||||
log_url_lookup[task['log_url']],
|
||||
task['job_guid'],
|
||||
push_timestamp_lookup[task['result_set_id']]
|
||||
],
|
||||
kwargs={'check_errors': task['check_errors']},
|
||||
routing_key=task['routing_key']
|
||||
)
|
||||
|
||||
def get_job_log_url_detail(self, job_log_url_id):
|
||||
obj = self.get_jobs_dhub().execute(
|
||||
proc='jobs.selects.get_job_log_url_detail',
|
||||
debug_show=self.DEBUG,
|
||||
placeholders=[job_log_url_id])
|
||||
if len(obj) == 0:
|
||||
raise ObjectNotFoundException("job_log_url", id=job_log_url_id)
|
||||
return obj[0]
|
||||
|
||||
def get_job_log_url_list(self, job_ids):
|
||||
"""
|
||||
Return a list of logs belonging to the given job_id(s).
|
||||
"""
|
||||
if len(job_ids) == 0:
|
||||
return []
|
||||
|
||||
replacement = []
|
||||
id_placeholders = ["%s"] * len(job_ids)
|
||||
replacement.append(','.join(id_placeholders))
|
||||
|
||||
data = self.get_jobs_dhub().execute(
|
||||
proc="jobs.selects.get_job_log_url_list",
|
||||
placeholders=job_ids,
|
||||
replace=replacement,
|
||||
debug_show=self.DEBUG,
|
||||
)
|
||||
return data
|
||||
|
||||
def update_job_log_url_status(self, job_log_url_id,
|
||||
parse_status, parse_timestamp):
|
||||
|
||||
self.get_jobs_dhub().execute(
|
||||
proc='jobs.updates.update_job_log_url',
|
||||
debug_show=self.DEBUG,
|
||||
placeholders=[parse_status, parse_timestamp, job_log_url_id])
|
||||
|
||||
|
||||
def store_job_artifact(self, artifact_placeholders):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -51,7 +51,6 @@
|
|||
},
|
||||
"inserts":{
|
||||
"create_job_data":{
|
||||
|
||||
"sql":"INSERT INTO `job` (
|
||||
`job_guid`,
|
||||
`signature`,
|
||||
|
|
@ -294,6 +293,14 @@
|
|||
",
|
||||
|
||||
"host":"master_host"
|
||||
},
|
||||
"update_job_log_url":{
|
||||
|
||||
"sql":"UPDATE `job_log_url`
|
||||
SET `parse_status` = ?, `parse_timestamp` = ?
|
||||
WHERE `id` = ?
|
||||
AND `active_status` = 'active'",
|
||||
"host":"master_host"
|
||||
}
|
||||
},
|
||||
"selects":{
|
||||
|
|
@ -589,6 +596,19 @@
|
|||
|
||||
"host":"read_host"
|
||||
},
|
||||
"get_job_log_url_detail":{
|
||||
"sql": "SELECT `id`, `job_id`, `name`, `url`, `parse_status`, `parse_timestamp`
|
||||
FROM job_log_url
|
||||
WHERE id = ? and active_status = 'active'",
|
||||
"host":"read_host"
|
||||
},
|
||||
"get_job_log_url_list":{
|
||||
"sql": "SELECT `id`, `job_id`, `name`, `url`, `parse_status`, `parse_timestamp`
|
||||
FROM job_log_url
|
||||
WHERE job_id in (REP0)
|
||||
and active_status='active'",
|
||||
"host": "read_host"
|
||||
},
|
||||
"get_job_artifact":{
|
||||
"sql":"SELECT
|
||||
id,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,92 @@
|
|||
from rest_framework import viewsets
|
||||
from rest_framework.response import Response
|
||||
from rest_framework.exceptions import ParseError
|
||||
from rest_framework.decorators import action
|
||||
from rest_framework.permissions import IsAuthenticated
|
||||
|
||||
from treeherder.webapp.api.utils import with_jobs, oauth_required
|
||||
|
||||
|
||||
class JobLogUrlViewSet(viewsets.ViewSet):
|
||||
"""
|
||||
A job_log_url object holds a reference to a job log.
|
||||
"""
|
||||
@with_jobs
|
||||
def retrieve(self, request, project, jm, pk=None):
|
||||
"""
|
||||
Returns a job_log_url object given its ID
|
||||
"""
|
||||
obj = jm.get_job_log_url_detail(pk)
|
||||
return Response(obj)
|
||||
|
||||
@with_jobs
|
||||
def list(self, request, project, jm):
|
||||
"""
|
||||
GET method implementation for list view
|
||||
job_id -- Mandatory filter indicating which job these log belongs to.
|
||||
"""
|
||||
|
||||
job_id = request.QUERY_PARAMS.get('job_id')
|
||||
if not job_id:
|
||||
raise ParseError(detail="The job_id parameter is mandatory for this endpoint")
|
||||
try:
|
||||
job_id = int(job_id)
|
||||
except ValueError:
|
||||
raise ParseError(detail="The job_id parameter must be an integer")
|
||||
|
||||
job_note_list = jm.get_job_log_url_list(job_id=job_id)
|
||||
return Response(job_note_list)
|
||||
|
||||
@with_jobs
|
||||
def list(self, request, project, jm):
|
||||
"""
|
||||
GET method implementation for list view
|
||||
job_id -- Mandatory filter indicating which job these log belongs to.
|
||||
"""
|
||||
|
||||
job_id = request.QUERY_PARAMS.get('job_id')
|
||||
if not job_id:
|
||||
raise ParseError(detail="The job_id parameter is mandatory for this endpoint")
|
||||
try:
|
||||
job_id = int(job_id)
|
||||
except ValueError:
|
||||
raise ParseError(detail="The job_id parameter must be an integer")
|
||||
|
||||
# get_job_log_url_list takes a lost of job ids
|
||||
job_log_url_list = jm.get_job_log_url_list([job_id])
|
||||
return Response(job_log_url_list)
|
||||
|
||||
@action()
|
||||
@with_jobs
|
||||
@oauth_required
|
||||
def update_parse_status(self, request, project, jm, pk=None):
|
||||
"""
|
||||
Change the state of a job.
|
||||
"""
|
||||
try:
|
||||
parse_status = request.DATA["parse_status"]
|
||||
parse_timestamp = request.DATA["parse_timestamp"]
|
||||
jm.update_job_log_url_status(pk, parse_status, parse_timestamp)
|
||||
obj = jm.get_job_log_url_detail(pk)
|
||||
return Response(obj)
|
||||
except KeyError:
|
||||
raise ParseError(detail=("The parse_status and parse_timestamp parameters"
|
||||
" are mandatory for this endpoint"))
|
||||
|
||||
@action(permission_classes=[IsAuthenticated])
|
||||
@with_jobs
|
||||
def parse(self, request, project, jm, pk=None):
|
||||
"""
|
||||
Trigger an async task to parse this log. This can be requested by the ui
|
||||
in case the log parsing had an intermittent failure
|
||||
"""
|
||||
log_obj = jm.get_job_log_url_detail(pk)
|
||||
job = jm.get_job(log_obj["job_id"])
|
||||
has_failed = job["result"] in jm.FAILED_RESULTS
|
||||
|
||||
from treeherder.log_parser.tasks import parse_log
|
||||
|
||||
parse_log.delay(project, log_obj["url"],
|
||||
job["job_guid"], job["resultset_id"],
|
||||
check_errors=has_failed)
|
||||
return Response({"message": "Log parsing triggered successfully"})
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
from django.conf.urls import patterns, include, url
|
||||
from treeherder.webapp.api import (refdata, objectstore, jobs, resultset,
|
||||
artifact, note, revision, bug, logslice)
|
||||
artifact, note, revision, bug, logslice,
|
||||
job_log_url)
|
||||
|
||||
from rest_framework import routers
|
||||
|
||||
|
|
@ -56,6 +57,12 @@ project_bound_router.register(
|
|||
base_name='logslice',
|
||||
)
|
||||
|
||||
project_bound_router.register(
|
||||
r'job-log-url',
|
||||
job_log_url.JobLogUrlViewSet,
|
||||
base_name='job-log-url',
|
||||
)
|
||||
|
||||
# this is the default router for plain restful endpoints
|
||||
|
||||
# refdata endpoints:
|
||||
|
|
|
|||
|
|
@ -6,7 +6,9 @@ from __future__ import unicode_literals
|
|||
import httplib
|
||||
import oauth2 as oauth
|
||||
import time
|
||||
import urllib
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
try:
|
||||
import json
|
||||
|
|
@ -219,11 +221,11 @@ class TreeherderJob(TreeherderData, ValidatorMixin):
|
|||
|
||||
def add_artifact(self, name, artifact_type, blob):
|
||||
if blob:
|
||||
self.data['job']['artifact'] = {
|
||||
'name': name,
|
||||
'type': artifact_type,
|
||||
'blob': blob
|
||||
}
|
||||
self.data['job']['artifacts'].append({
|
||||
'name': name,
|
||||
'type': artifact_type,
|
||||
'blob': blob
|
||||
})
|
||||
|
||||
def init_data(self):
|
||||
|
||||
|
|
@ -315,11 +317,7 @@ class TreeherderJob(TreeherderData, ValidatorMixin):
|
|||
# project_jobs_1.job_artifact.name
|
||||
# project_jobs_1.job_artifact.type
|
||||
# project_jobs_1.job_artifact.blob
|
||||
'artifact': {
|
||||
'name': '',
|
||||
'type': '',
|
||||
'blob': ''
|
||||
}
|
||||
'artifacts': []
|
||||
},
|
||||
|
||||
# List of job_guids that were coallesced to this job
|
||||
|
|
@ -411,11 +409,11 @@ class TreeherderResultSet(TreeherderData, ValidatorMixin):
|
|||
'type': '',
|
||||
# TODO: add resultset artifact table in treeherder-service
|
||||
'artifact': {
|
||||
'type': '',
|
||||
'name': '',
|
||||
'blob': ''
|
||||
}
|
||||
'name': "",
|
||||
'type': "",
|
||||
'blob': ""
|
||||
}
|
||||
}
|
||||
|
||||
def add_push_timestamp(self, push_timestamp):
|
||||
self.data['push_timestamp'] = push_timestamp
|
||||
|
|
@ -443,10 +441,10 @@ class TreeherderResultSet(TreeherderData, ValidatorMixin):
|
|||
def add_artifact(self, name, artifact_type, blob):
|
||||
if blob:
|
||||
self.data['artifact'] = {
|
||||
'name': name,
|
||||
'type': artifact_type,
|
||||
'blob': blob
|
||||
}
|
||||
'name': name,
|
||||
'type': artifact_type,
|
||||
'blob': blob
|
||||
}
|
||||
|
||||
def get_revision(self, data={}):
|
||||
return TreeherderRevision(data)
|
||||
|
|
@ -585,6 +583,47 @@ class TreeherderArtifactCollection(TreeherderCollection):
|
|||
return TreeherderArtifact(data)
|
||||
|
||||
|
||||
class OauthClient(object):
|
||||
"""
|
||||
A utility class containing the logic to sign a oauth request
|
||||
"""
|
||||
def __init__(self, oauth_key, oauth_secret, user):
|
||||
self.oauth_key = oauth_key
|
||||
self.oauth_secret = oauth_secret
|
||||
self.user = user
|
||||
|
||||
def get_signed_uri(self, serialized_body, uri, http_method):
|
||||
|
||||
# There is no requirement for the token in two-legged
|
||||
# OAuth but we still need the token object.
|
||||
token = oauth.Token(key='', secret='')
|
||||
consumer = oauth.Consumer(key=self.oauth_key, secret=self.oauth_secret)
|
||||
|
||||
parameters = {
|
||||
'user': self.user,
|
||||
'oauth_version': '1.0',
|
||||
'oauth_nonce': oauth.generate_nonce(),
|
||||
'oauth_timestamp': int(time.time())
|
||||
}
|
||||
|
||||
try:
|
||||
req = oauth.Request(
|
||||
method=http_method,
|
||||
body=serialized_body,
|
||||
url=uri,
|
||||
parameters=parameters
|
||||
)
|
||||
except AssertionError, e:
|
||||
logger.error('uri: %s' % uri)
|
||||
logger.error('body: %s' % serialized_body)
|
||||
raise
|
||||
|
||||
signature_method = oauth.SignatureMethod_HMAC_SHA1()
|
||||
req.sign_request(signature_method, consumer, token)
|
||||
|
||||
return req.to_url()
|
||||
|
||||
|
||||
class TreeherderRequest(object):
|
||||
"""
|
||||
Treeherder request object that manages test submission.
|
||||
|
|
@ -604,6 +643,10 @@ class TreeherderRequest(object):
|
|||
self.project = project
|
||||
self.oauth_key = oauth_key
|
||||
self.oauth_secret = oauth_secret
|
||||
self.use_oauth = bool(self.oauth_key and self.oauth_secret)
|
||||
self.oauth_client = None
|
||||
if self.use_oauth:
|
||||
self.oauth_client = OauthClient(oauth_key, oauth_secret, self.project)
|
||||
|
||||
if protocol not in self.protocols:
|
||||
raise AssertionError('Protocol "%s" not supported; please use one of %s' %
|
||||
|
|
@ -615,9 +658,11 @@ class TreeherderRequest(object):
|
|||
msg = "{0}: project required for posting".format(self.__class__.__name__)
|
||||
raise TreeherderClientError(msg, [])
|
||||
|
||||
def send(self, collection_inst):
|
||||
"""
|
||||
Send given treeherder collection instance data to server; returns httplib Response.
|
||||
def post(self, collection_inst):
|
||||
"""Shortcut method to send a treeherder collection via POST
|
||||
|
||||
:param collection_inst: a TreeherderCollection instance
|
||||
:returns: an httplib Response object
|
||||
"""
|
||||
|
||||
if not isinstance(collection_inst, TreeherderCollection):
|
||||
|
|
@ -642,65 +687,58 @@ class TreeherderRequest(object):
|
|||
|
||||
collection_inst.validate()
|
||||
|
||||
return self.send(collection_inst.endpoint_base,
|
||||
method="POST",
|
||||
data=collection_inst.to_json())
|
||||
|
||||
def send(self, endpoint, method=None, data=None):
|
||||
"""send data to the given endpoint with the given http method.
|
||||
|
||||
:param endpoint: the target endpoint for this request
|
||||
:param method: can be one of GET,POST,PUT
|
||||
:param data: the body of this request
|
||||
:returns: an httplib Response object
|
||||
"""
|
||||
|
||||
if method not in ("GET", "POST", "PUT"):
|
||||
msg = "{0}: {1} is not a supported method".format(
|
||||
self.__class__.__name__,
|
||||
method
|
||||
)
|
||||
raise TreeherderClientError(msg, [])
|
||||
|
||||
# Build the header
|
||||
headers = {'Content-Type': 'application/json'}
|
||||
|
||||
use_oauth = bool(self.oauth_key and self.oauth_secret)
|
||||
if data:
|
||||
if not isinstance(data, str):
|
||||
# if the body is not serialized yet, do it now
|
||||
serialized_body = json.dumps(data)
|
||||
else:
|
||||
serialized_body = data
|
||||
else:
|
||||
serialized_body = None
|
||||
|
||||
serialized_body = collection_inst.to_json()
|
||||
uri = self.get_uri(endpoint)
|
||||
|
||||
uri = self.get_uri(collection_inst)
|
||||
if self.use_oauth:
|
||||
uri = self.oauth_client.get_signed_uri(serialized_body, uri, method)
|
||||
|
||||
if use_oauth:
|
||||
uri = self.get_signed_uri(serialized_body, uri)
|
||||
|
||||
# Make the POST request
|
||||
# Make the request
|
||||
conn = None
|
||||
if self.protocol == 'http':
|
||||
conn = httplib.HTTPConnection(self.host)
|
||||
else:
|
||||
conn = httplib.HTTPSConnection(self.host)
|
||||
|
||||
conn.request('POST', uri, serialized_body, headers)
|
||||
conn.request(method, uri, serialized_body, headers)
|
||||
|
||||
return conn.getresponse()
|
||||
|
||||
def get_signed_uri(self, serialized_body, uri):
|
||||
|
||||
# There is no requirement for the token in two-legged
|
||||
# OAuth but we still need the token object.
|
||||
token = oauth.Token(key='', secret='')
|
||||
consumer = oauth.Consumer(key=self.oauth_key, secret=self.oauth_secret)
|
||||
|
||||
parameters = {
|
||||
'user':self.project,
|
||||
'oauth_version':'1.0',
|
||||
'oauth_nonce':oauth.generate_nonce(),
|
||||
'oauth_timestamp':int(time.time())
|
||||
}
|
||||
|
||||
try:
|
||||
req = oauth.Request(
|
||||
method='POST',
|
||||
body=serialized_body,
|
||||
url=uri,
|
||||
parameters=parameters
|
||||
)
|
||||
except AssertionError, e:
|
||||
print 'uri: %s' % uri
|
||||
print 'body: %s' % serialized_body
|
||||
raise
|
||||
|
||||
signature_method = oauth.SignatureMethod_HMAC_SHA1()
|
||||
req.sign_request(signature_method, consumer, token)
|
||||
|
||||
return req.to_url()
|
||||
|
||||
def get_uri(self, collection_inst):
|
||||
def get_uri(self, endpoint):
|
||||
|
||||
uri = '{0}://{1}/api/project/{2}/{3}/'.format(
|
||||
self.protocol, self.host, self.project, collection_inst.endpoint_base
|
||||
self.protocol, self.host, self.project, endpoint
|
||||
)
|
||||
|
||||
return uri
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче