зеркало из https://github.com/mozilla/treeherder.git
432 строки
14 KiB
Python
432 строки
14 KiB
Python
import json
|
|
|
|
import pytest
|
|
from mock import MagicMock
|
|
|
|
from tests.test_utils import post_collection
|
|
from treeherder.client.thclient import client
|
|
from treeherder.log_parser.parsers import StepParser
|
|
from treeherder.model import error_summary
|
|
from treeherder.model.derived import (ArtifactsModel,
|
|
JobsModel)
|
|
|
|
|
|
@pytest.fixture
|
|
def text_log_summary_dict():
|
|
return {
|
|
"step_data": {
|
|
"all_errors": [
|
|
{"line": "12:34:13 INFO - Assertion failure: addr % CellSize == 0, at ../../../js/src/gc/Heap.h:1041", "linenumber": 61918},
|
|
{"line": "12:34:24 WARNING - TEST-UNEXPECTED-FAIL | file:///builds/slave/talos-slave/test/build/tests/jsreftest/tests/jsreftest.html?test=ecma_5/JSON/parse-array-gc.js | Exited with code 1 during test run", "linenumber": 61919}, {"line": "12:34:37 WARNING - PROCESS-CRASH | file:///builds/slave/talos-slave/test/build/tests/jsreftest/tests/jsreftest.html?test=ecma_5/JSON/parse-array-gc.js | application crashed [@ js::gc::Cell::tenuredZone() const]", "linenumber": 61922},
|
|
{"line": "12:34:38 ERROR - Return code: 256", "linenumber": 64435}
|
|
],
|
|
"steps": [
|
|
{"name": "Clone gecko tc-vcs "},
|
|
{"name": "Build ./build-b2g-desktop.sh /home/worker/workspace"}
|
|
],
|
|
"errors_truncated": False
|
|
},
|
|
"logurl": "https://queue.taskcluster.net/v1/task/nhxC4hC3RE6LSVWTZT4rag/runs/0/artifacts/public/logs/live_backing.log"
|
|
}
|
|
|
|
|
|
def check_artifacts(test_project,
|
|
job_guid,
|
|
parse_status,
|
|
num_artifacts,
|
|
exp_artifact_names=None,
|
|
exp_error_summary=None):
|
|
|
|
with JobsModel(test_project) as jobs_model:
|
|
job_id = [x['id'] for x in jobs_model.get_job_list(0, 20)
|
|
if x['job_guid'] == job_guid][0]
|
|
job_log_list = jobs_model.get_job_log_url_list([job_id])
|
|
|
|
assert len(job_log_list) == 1
|
|
assert job_log_list[0]['parse_status'] == parse_status
|
|
|
|
with ArtifactsModel(test_project) as artifacts_model:
|
|
artifacts = artifacts_model.get_job_artifact_list(0, 10, conditions={
|
|
'job_id': {('=', job_id)}
|
|
})
|
|
|
|
assert len(artifacts) == num_artifacts
|
|
|
|
if exp_artifact_names:
|
|
artifact_names = {x['name'] for x in artifacts}
|
|
assert set(artifact_names) == exp_artifact_names
|
|
|
|
if exp_error_summary:
|
|
act_bs_obj = [x['blob'] for x in artifacts if x['name'] == 'Bug suggestions'][0]
|
|
assert exp_error_summary == act_bs_obj
|
|
|
|
|
|
def test_post_job_with_parsed_log(test_project, result_set_stored,
|
|
mock_post_json,
|
|
monkeypatch,
|
|
):
|
|
"""
|
|
test submitting a job with a pre-parsed log gets job_log_url
|
|
parse_status of "parsed" and does not parse, even though no text_log_summary
|
|
exists.
|
|
|
|
This is for the case where they may want to submit it at a later time.
|
|
"""
|
|
|
|
mock_parse = MagicMock(name="parse_line")
|
|
monkeypatch.setattr(StepParser, 'parse_line', mock_parse)
|
|
|
|
tjc = client.TreeherderJobCollection()
|
|
job_guid = 'd22c74d4aa6d2a1dcba96d95dccbd5fdca70cf33'
|
|
tj = client.TreeherderJob({
|
|
'project': test_project,
|
|
'revision_hash': result_set_stored[0]['revision_hash'],
|
|
'job': {
|
|
'job_guid': job_guid,
|
|
'state': 'completed',
|
|
'log_references': [{
|
|
'url': 'http://ftp.mozilla.org/pub/mozilla.org/spidermonkey/...',
|
|
'name': 'buildbot_text',
|
|
'parse_status': 'parsed'
|
|
}]
|
|
}
|
|
})
|
|
tjc.add(tj)
|
|
|
|
post_collection(test_project, tjc)
|
|
|
|
check_artifacts(test_project, job_guid, 'parsed', 0)
|
|
|
|
# ensure the parsing didn't happen
|
|
assert mock_parse.called is False
|
|
|
|
|
|
def test_post_job_with_text_log_summary_artifact_parsed(
|
|
test_project,
|
|
monkeypatch,
|
|
result_set_stored,
|
|
mock_post_json,
|
|
mock_error_summary,
|
|
text_log_summary_dict,
|
|
):
|
|
"""
|
|
test submitting a job with a pre-parsed log gets parse_status of
|
|
"parsed" and doesn't parse the log, but still generates
|
|
the bug suggestions.
|
|
"""
|
|
|
|
mock_parse = MagicMock(name="parse_line")
|
|
monkeypatch.setattr(StepParser, 'parse_line', mock_parse)
|
|
|
|
job_guid = 'd22c74d4aa6d2a1dcba96d95dccbd5fdca70cf33'
|
|
tjc = client.TreeherderJobCollection()
|
|
tj = client.TreeherderJob({
|
|
'project': test_project,
|
|
'revision_hash': result_set_stored[0]['revision_hash'],
|
|
'job': {
|
|
'job_guid': job_guid,
|
|
'state': 'completed',
|
|
'log_references': [{
|
|
'url': 'http://ftp.mozilla.org/pub/mozilla.org/spidermonkey/...',
|
|
'name': 'buildbot_text',
|
|
'parse_status': 'parsed'
|
|
}],
|
|
'artifacts': [{
|
|
"blob": json.dumps(text_log_summary_dict),
|
|
"type": "json",
|
|
"name": "text_log_summary",
|
|
"job_guid": job_guid
|
|
}]
|
|
}
|
|
})
|
|
tjc.add(tj)
|
|
|
|
post_collection(test_project, tjc)
|
|
|
|
check_artifacts(test_project, job_guid, 'parsed', 2,
|
|
{'Bug suggestions', 'text_log_summary'}, mock_error_summary)
|
|
|
|
# ensure the parsing didn't happen
|
|
assert mock_parse.called is False
|
|
|
|
|
|
def test_post_job_with_text_log_summary_artifact_parsed_dict_blob(
|
|
test_project,
|
|
monkeypatch,
|
|
result_set_stored,
|
|
mock_post_json,
|
|
mock_error_summary,
|
|
text_log_summary_dict,
|
|
):
|
|
"""
|
|
test submitting a job with a pre-parsed log gets parse_status of
|
|
"parsed" and doesn't parse the log, but still generates
|
|
the bug suggestions.
|
|
"""
|
|
|
|
mock_parse = MagicMock(name="parse_line")
|
|
monkeypatch.setattr(StepParser, 'parse_line', mock_parse)
|
|
|
|
job_guid = 'd22c74d4aa6d2a1dcba96d95dccbd5fdca70cf33'
|
|
tjc = client.TreeherderJobCollection()
|
|
tj = client.TreeherderJob({
|
|
'project': test_project,
|
|
'revision_hash': result_set_stored[0]['revision_hash'],
|
|
'job': {
|
|
'job_guid': job_guid,
|
|
'state': 'completed',
|
|
'log_references': [{
|
|
'url': 'http://ftp.mozilla.org/pub/mozilla.org/spidermonkey/...',
|
|
'name': 'buildbot_text',
|
|
'parse_status': 'parsed'
|
|
}],
|
|
'artifacts': [{
|
|
"blob": text_log_summary_dict,
|
|
"type": "json",
|
|
"name": "text_log_summary",
|
|
"job_guid": job_guid
|
|
}]
|
|
}
|
|
})
|
|
tjc.add(tj)
|
|
|
|
post_collection(test_project, tjc)
|
|
|
|
check_artifacts(test_project, job_guid, 'parsed', 2,
|
|
{'Bug suggestions', 'text_log_summary'}, mock_error_summary)
|
|
|
|
# ensure the parsing didn't happen
|
|
assert mock_parse.called is False
|
|
|
|
|
|
def test_post_job_with_text_log_summary_artifact_pending(
|
|
test_project,
|
|
monkeypatch,
|
|
result_set_stored,
|
|
mock_post_json,
|
|
mock_error_summary,
|
|
text_log_summary_dict,
|
|
):
|
|
"""
|
|
test submitting a job with a log set to pending, but with a text_log_summary.
|
|
|
|
This should detect the artifact, not parse, and just mark the log as parsed,
|
|
then generate bug suggestions.
|
|
"""
|
|
|
|
mock_parse = MagicMock(name="parse_line")
|
|
monkeypatch.setattr(StepParser, 'parse_line', mock_parse)
|
|
|
|
job_guid = 'd22c74d4aa6d2a1dcba96d95dccbd5fdca70cf33'
|
|
tjc = client.TreeherderJobCollection()
|
|
tj = client.TreeherderJob({
|
|
'project': test_project,
|
|
'revision_hash': result_set_stored[0]['revision_hash'],
|
|
'job': {
|
|
'job_guid': job_guid,
|
|
'state': 'completed',
|
|
'log_references': [{
|
|
'url': 'http://ftp.mozilla.org/pub/mozilla.org/spidermonkey/...',
|
|
'name': 'buildbot_text',
|
|
'parse_status': 'pending'
|
|
}],
|
|
'artifacts': [{
|
|
"blob": json.dumps(text_log_summary_dict),
|
|
"type": "json",
|
|
"name": "text_log_summary",
|
|
"job_guid": job_guid
|
|
}]
|
|
}
|
|
})
|
|
|
|
tjc.add(tj)
|
|
|
|
post_collection(test_project, tjc)
|
|
|
|
check_artifacts(test_project, job_guid, 'parsed', 2,
|
|
{'Bug suggestions', 'text_log_summary'}, mock_error_summary)
|
|
|
|
# ensure the parsing didn't happen
|
|
assert mock_parse.called is False
|
|
|
|
|
|
def test_post_job_with_text_log_summary_and_bug_suggestions_artifact(
|
|
test_project,
|
|
monkeypatch,
|
|
result_set_stored,
|
|
mock_post_json,
|
|
mock_error_summary,
|
|
text_log_summary_dict,
|
|
):
|
|
"""
|
|
test submitting a job with a pre-parsed log and both artifacts
|
|
does not generate parse the log or generate any artifacts, just uses
|
|
the supplied ones.
|
|
"""
|
|
|
|
mock_parse = MagicMock(name="parse_line")
|
|
monkeypatch.setattr(StepParser, 'parse_line', mock_parse)
|
|
mock_get_error_summary = MagicMock(name="get_error_summary_artifacts")
|
|
monkeypatch.setattr(error_summary, 'get_error_summary_artifacts', mock_get_error_summary)
|
|
|
|
error_summary_blob = ["fee", "fie", "foe", "fum"]
|
|
|
|
job_guid = 'd22c74d4aa6d2a1dcba96d95dccbd5fdca70cf33'
|
|
tjc = client.TreeherderJobCollection()
|
|
tj = client.TreeherderJob({
|
|
'project': test_project,
|
|
'revision_hash': result_set_stored[0]['revision_hash'],
|
|
'job': {
|
|
'job_guid': job_guid,
|
|
'state': 'completed',
|
|
'log_references': [{
|
|
'url': 'http://ftp.mozilla.org/pub/mozilla.org/spidermonkey/...',
|
|
'name': 'buildbot_text',
|
|
'parse_status': 'parsed'
|
|
}],
|
|
'artifacts': [
|
|
{
|
|
"blob": json.dumps(text_log_summary_dict),
|
|
"type": "json",
|
|
"name": "text_log_summary",
|
|
"job_guid": job_guid
|
|
},
|
|
{
|
|
"blob": json.dumps(error_summary_blob),
|
|
"type": "json",
|
|
"name": "Bug suggestions",
|
|
"job_guid": job_guid
|
|
},
|
|
]
|
|
}
|
|
})
|
|
|
|
tjc.add(tj)
|
|
|
|
post_collection(test_project, tjc)
|
|
|
|
check_artifacts(test_project, job_guid, 'parsed', 2,
|
|
{'Bug suggestions', 'text_log_summary'}, error_summary_blob)
|
|
|
|
assert mock_parse.called is False
|
|
assert mock_get_error_summary.called is False
|
|
|
|
|
|
def test_post_job_artifacts_by_add_artifact(
|
|
test_project,
|
|
monkeypatch,
|
|
result_set_stored,
|
|
mock_post_json,
|
|
mock_error_summary,
|
|
):
|
|
"""
|
|
test submitting a job with artifacts added by ``add_artifact``
|
|
|
|
This has pre-parsed logs. Verify parse_status of "parsed" and that it
|
|
doesn't parse the logs.
|
|
|
|
Submitted ``text_log_artifact`` should still trigger generation of the
|
|
bug suggestions.
|
|
"""
|
|
|
|
mock_parse = MagicMock(name="parse_line")
|
|
monkeypatch.setattr(StepParser, 'parse_line', mock_parse)
|
|
|
|
job_guid = 'd22c74d4aa6d2a1dcba96d95dccbd5fdca70cf33'
|
|
tjc = client.TreeherderJobCollection()
|
|
tj = client.TreeherderJob({
|
|
'project': test_project,
|
|
'revision_hash': result_set_stored[0]['revision_hash'],
|
|
"job": {
|
|
"artifacts": [],
|
|
"job_guid": job_guid,
|
|
"log_references": [
|
|
{
|
|
"name": "autophone-nexus-one-1.log",
|
|
"parse_status": "parsed",
|
|
"url": "https://autophone-dev.s3.amazonaws.com/pub/mozilla.org/mobile/tinderbox-builds/mozilla-inbound-android-api-9/1432676531/en-US/autophone-autophone-s1s2-s1s2-nytimes-local.ini-1-nexus-one-1.log"
|
|
}
|
|
],
|
|
"state": "completed",
|
|
},
|
|
})
|
|
|
|
tls_blob = json.dumps({
|
|
"logurl": "https://autophone-dev.s3.amazonaws.com/pub/mozilla.org/mobile/tinderbox-builds/mozilla-inbound-android-api-9/1432676531/en-US/autophone-autophone-s1s2-s1s2-nytimes-local.ini-1-nexus-one-1.log",
|
|
"step_data": {
|
|
"all_errors": [
|
|
{"line": "TEST_UNEXPECTED_FAIL | /sdcard/tests/autophone/s1s2test/nytimes.com/index.html | Failed to get uncached measurement.", "linenumber": 64435},
|
|
],
|
|
"steps": [{"buncha": "info"}]
|
|
}
|
|
})
|
|
|
|
ji_blob = json.dumps({"job_details": [{"foo": "fah"}]})
|
|
bapi_blob = json.dumps({"buildername": "merd"})
|
|
pb_blob = json.dumps({"build_url": "feh", "chunk": 1, "config_file": "mah"})
|
|
|
|
tj.add_artifact("text_log_summary", "json", tls_blob)
|
|
tj.add_artifact("Job Info", "json", ji_blob)
|
|
tj.add_artifact("buildapi", "json", bapi_blob)
|
|
tj.add_artifact("privatebuild", "json", pb_blob)
|
|
|
|
tjc.add(tj)
|
|
|
|
post_collection(test_project, tjc)
|
|
|
|
check_artifacts(test_project, job_guid, 'parsed', 5,
|
|
{'Bug suggestions', 'text_log_summary', 'Job Info',
|
|
'privatebuild', 'buildapi'}, mock_error_summary)
|
|
|
|
# ensure the parsing didn't happen
|
|
assert mock_parse.called is False
|
|
|
|
|
|
def test_post_job_with_tier(test_project, result_set_stored,
|
|
mock_post_json):
|
|
"""test submitting a job with tier specified"""
|
|
|
|
tjc = client.TreeherderJobCollection()
|
|
job_guid = 'd22c74d4aa6d2a1dcba96d95dccbd5fdca70cf33'
|
|
tj = client.TreeherderJob({
|
|
'project': test_project,
|
|
'revision_hash': result_set_stored[0]['revision_hash'],
|
|
'job': {
|
|
'job_guid': job_guid,
|
|
'state': 'completed',
|
|
}
|
|
})
|
|
tj.add_tier(3)
|
|
tjc.add(tj)
|
|
|
|
post_collection(test_project, tjc)
|
|
|
|
with JobsModel(test_project) as jobs_model:
|
|
job = [x for x in jobs_model.get_job_list(0, 20)
|
|
if x['job_guid'] == job_guid][0]
|
|
assert job['tier'] == 3
|
|
|
|
|
|
def test_post_job_with_default_tier(test_project, result_set_stored,
|
|
mock_post_json):
|
|
"""test submitting a job with no tier specified gets default"""
|
|
|
|
tjc = client.TreeherderJobCollection()
|
|
job_guid = 'd22c74d4aa6d2a1dcba96d95dccbd5fdca70cf33'
|
|
tj = client.TreeherderJob({
|
|
'project': test_project,
|
|
'revision_hash': result_set_stored[0]['revision_hash'],
|
|
'job': {
|
|
'job_guid': job_guid,
|
|
'state': 'completed',
|
|
}
|
|
})
|
|
tjc.add(tj)
|
|
|
|
post_collection(test_project, tjc)
|
|
|
|
with JobsModel(test_project) as jobs_model:
|
|
job = [x for x in jobs_model.get_job_list(0, 20)
|
|
if x['job_guid'] == job_guid][0]
|
|
assert job['tier'] == 1
|