mozilla
/
treeherder
зеркало из https://github.com/mozilla/treeherder.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Remove step parser v3 (#6634) 

* remove StepParser and switch to ErrorParser
* remove writes to TextLogStep from artifact.py
* remove buildbot ref in builders
* replace TextLogStep model in DetailsPanel, SimilarJobsTab and logviewer App
* cleanup DetailsPanel
* remove old log parsing tests and update others
* add logging to error_summary.py
* add parse max error lines limit to ErrorParser
* fix in similar jobs tab for Bug 1652869
This commit is contained in:
Sarah Clements 2020-07-17 10:28:01 -07:00 коммит произвёл GitHub
Родитель 10ccd44545
Коммит 507881a7b8
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
47 изменённых файлов: 526 добавлений и 2534 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

83
schemas/text-log-summary-artifact.yml
Просмотреть файл

@ -1,83 +0,0 @@
'$schema': http://json-schema.org/draft-04/schema#
type: object
properties:
blob:
type: object
properties:
step_data:
type: object
properties:
steps:
type: array
items:
type: object
properties:
errors:
type: array
items:
type: object
properties:
line:
type: string
linenumber:
type: integer
additionalProperties: false
additionalItems: false
name:
type: string
started:
type: string
format: date-time
started_linenumber:
type: integer
finished_linenumber:
type: integer
finished:
type: string
format: date-time
result:
type: string
enum:
[
busted,
testfailed,
exception,
success,
canceled,
unknown,
retry,
skipped,
]
required:
- errors
- name
- started
- started_linenumber
- finished_linenumber
- finished
- result
additionalProperties: false
additionalItems: false
errors_truncated:
type: boolean
additionalProperties: false
required:
- steps
- errors_truncated
logurl:
type: string
additionalProperties: false
required:
- step_data
- logurl
type:
type: string
name:
type: string
job_guid:
type: integer
additionalProperties: false
required:
- blob
- type
- name

4
tests/autoclassify/test_classify_failures.py
Просмотреть файл

@ -104,9 +104,7 @@ def test_autoclassify_no_update_job_classification(
lines = [(test_line, {})]
test_error_lines, test_failure_lines = create_lines(test_job_2, lines)
TextLogError.objects.create(
step=test_error_lines[0].step,
line="Some error that isn't in the structured logs",
line_number=2,
job=test_job_2, line="Some error that isn't in the structured logs", line_number=2,
)
do_autoclassify(test_job_2, test_failure_lines, [precise_matcher])

15
tests/autoclassify/utils.py
Просмотреть файл

@ -1,5 +1,3 @@
import datetime
from mozlog.formatters.tbplformatter import TbplFormatter
from treeherder.model.models import (
@ -8,7 +6,6 @@ from treeherder.model.models import (
JobLog,
TextLogError,
TextLogErrorMetadata,
TextLogStep,
)
test_line = {
@ -79,16 +76,6 @@ def get_data(base_data, updates):
def create_text_log_errors(job, failure_line_list):
step = TextLogStep.objects.create(
job=job,
name='everything',
started_line_number=1,
finished_line_number=10,
started=datetime.datetime.now(),
finished=datetime.datetime.now(),
result=TextLogStep.TEST_FAILED,
)
formatter = TbplFormatter()
errors = []
for i, (base_data, updates) in enumerate(failure_line_list):
@ -96,7 +83,7 @@ def create_text_log_errors(job, failure_line_list):
if not data:
continue
error = TextLogError.objects.create(
step=step, line=formatter(data).split("\n")[0], line_number=i
job=job, line=formatter(data).split("\n")[0], line_number=i
)
errors.append(error)

38
tests/etl/test_load_artifacts.py
Просмотреть файл

@ -1,7 +1,7 @@
import json
from treeherder.etl.artifact import store_job_artifacts
from treeherder.model.models import TextLogError, TextLogStep
from treeherder.model.models import TextLogError
def test_load_textlog_summary_twice(test_repository, test_job):
@ -10,32 +10,21 @@ def test_load_textlog_summary_twice(test_repository, test_job):
'name': 'text_log_summary',
'blob': json.dumps(
{
'step_data': {
"steps": [
{
'name': 'foo',
'started': '2016-05-10 12:44:23.103904',
'started_linenumber': 8,
'finished_linenumber': 10,
'finished': '2016-05-10 12:44:23.104394',
'result': 'success',
'errors': [{"line": '07:51:28 WARNING - foobar', "linenumber": 1587}],
}
]
}
'errors': [
{"line": 'WARNING - foobar', "linenumber": 1587},
{"line": 'WARNING - foobar', "linenumber": 1590},
],
}
),
'job_guid': test_job.guid,
}
store_job_artifacts([text_log_summary_artifact])
assert TextLogError.objects.count() == 1
assert TextLogStep.objects.count() == 1
assert TextLogError.objects.count() == 2
# load again (simulating the job being parsed twice,
# which sometimes happens)
store_job_artifacts([text_log_summary_artifact])
assert TextLogError.objects.count() == 1
assert TextLogStep.objects.count() == 1
assert TextLogError.objects.count() == 2
def test_load_non_ascii_textlog_errors(test_job):
@ -44,15 +33,6 @@ def test_load_non_ascii_textlog_errors(test_job):
'name': 'text_log_summary',
'blob': json.dumps(
{
'step_data': {
"steps": [
{
'name': 'foo',
'started': '2016-05-10 12:44:23.103904',
'started_linenumber': 8,
'finished_linenumber': 10,
'finished': '2016-05-10 12:44:23.104394',
'result': 'success',
'errors': [
{
# non-ascii character
@ -66,12 +46,10 @@ def test_load_non_ascii_textlog_errors(test_job):
},
],
}
]
}
}
),
'job_guid': test_job.guid,
}
store_job_artifacts([text_log_summary_artifact])
assert TextLogError.objects.count() == 2

8
tests/log_parser/test_artifact_builder_collection.py
Просмотреть файл

@ -7,19 +7,19 @@ from treeherder.log_parser.artifactbuildercollection import (
ArtifactBuilderCollection,
LogSizeException,
)
from treeherder.log_parser.artifactbuilders import BuildbotLogViewArtifactBuilder
from treeherder.log_parser.artifactbuilders import LogViewerArtifactBuilder
def test_builders_as_list():
"""test that passing in a list of builders works"""
builder = BuildbotLogViewArtifactBuilder()
builder = LogViewerArtifactBuilder()
lpc = ArtifactBuilderCollection("foo-url", builders=[builder])
assert lpc.builders == [builder]
def test_builders_as_single_still_list():
"""test that passing in a single builder becomes a list"""
builder = BuildbotLogViewArtifactBuilder()
builder = LogViewerArtifactBuilder()
lpc = ArtifactBuilderCollection("foo-url", builders=builder)
assert lpc.builders == [builder]
@ -43,7 +43,7 @@ def test_all_builders_complete():
lpc.parse()
exp = {
"text_log_summary": {"step_data": {"steps": [], "errors_truncated": False}, "logurl": url,},
"text_log_summary": {"errors": [], "logurl": url,},
}
assert exp == lpc.artifacts

90
tests/log_parser/test_log_view_artifact_builder.py
Просмотреть файл

@ -4,9 +4,9 @@ import responses
from tests import test_utils
from tests.test_utils import add_log_response
from treeherder.log_parser.artifactbuildercollection import ArtifactBuilderCollection
from treeherder.log_parser.artifactbuilders import BuildbotLogViewArtifactBuilder
from treeherder.log_parser.artifactbuilders import LogViewerArtifactBuilder
slow = pytest.mark.slow
skip = pytest.mark.skip
@responses.activate
@ -20,7 +20,7 @@ def do_test(log):
url = add_log_response("{}.txt.gz".format(log))
builder = BuildbotLogViewArtifactBuilder(url)
builder = LogViewerArtifactBuilder(url)
lpc = ArtifactBuilderCollection(url, builders=builder)
lpc.parse()
act = lpc.artifacts[builder.name]
@ -36,23 +36,6 @@ def do_test(log):
assert act == exp
def test_crashtest_passing():
"""Process a job with a single log reference."""
do_test("mozilla-central_fedora-b2g_test-crashtest-1-bm54-tests1-linux-build50")
def test_mochitest_pass():
"""Process a job with a single log reference."""
do_test("mozilla-central_mountainlion_test-mochitest-2-bm77-tests1-macosx-build141")
def test_duration_gt_1hr():
do_test("mozilla-central-win32-pgo-bm85-build1-build111")
@slow
def test_mochitest_fail():
"""Process a job with a single log reference."""
@ -65,128 +48,133 @@ def test_mochitest_process_crash():
do_test("mozilla-inbound_ubuntu64_vm-debug_test-mochitest-other-bm53-tests1-linux-build122")
@slow
def test_crash_1():
"""Test from old log parser"""
do_test("crash-1")
def test_opt_objc_exception():
"""Test from old log parser"""
do_test("opt-objc-exception")
def test_jsreftest_fail():
"""Test from old log parser"""
do_test("jsreftest-fail")
# TODO remove old tests and update active tests with current live backing logs
@skip
def test_jetpack_fail():
"""Process a job with a single log reference."""
do_test("ux_ubuntu32_vm_test-jetpack-bm67-tests1-linux-build16")
@slow
def test_crash_1():
"""Test from old log parser"""
do_test("crash-1")
@slow
@skip
def test_crash_2():
"""Test from old log parser"""
do_test("crash-2")
@slow
@skip
def test_crash_mac_1():
"""Test from old log parser"""
do_test("crash-mac-1")
@slow
@skip
def test_crashtest_timeout():
"""Test from old log parser"""
do_test("crashtest-timeout")
@slow
def test_jsreftest_fail():
"""Test from old log parser"""
do_test("jsreftest-fail")
@slow
@skip
def test_jsreftest_timeout_crash():
"""Test from old log parser"""
do_test("jsreftest-timeout-crash")
@slow
@skip
def test_leaks_1():
"""Test from old log parser"""
do_test("leaks-1")
@slow
@skip
def test_mochitest_test_end():
"""Test from old log parser"""
do_test("mochitest-test-end")
@slow
@skip
def test_multiple_timeouts():
"""Test from old log parser"""
do_test("multiple-timeouts")
@slow
def test_opt_objc_exception():
"""Test from old log parser"""
do_test("opt-objc-exception")
@slow
@skip
def test_reftest_fail_crash():
"""Test from old log parser"""
do_test("reftest-fail-crash")
@slow
@skip
def test_reftest_jserror():
"""Test from old log parser"""
do_test("reftest-jserror")
@slow
@skip
def test_reftest_opt_fail():
"""Test from old log parser"""
do_test("reftest-opt-fail")
@slow
@skip
def test_reftest_timeout():
"""Test from old log parser"""
do_test("reftest-timeout")
@slow
@skip
def test_tinderbox_exception():
"""Test from old log parser"""
do_test("tinderbox-exception")
@skip
def test_xpcshell_crash():
"""Test from old log parser"""
do_test("xpcshell-crash")
@skip
def test_xpcshell_multiple():
"""Test from old log parser"""
do_test("xpcshell-multiple")
@skip
def test_xpcshell_timeout():
"""Test from old log parser"""
do_test("xpcshell-timeout")
@skip
# This test is not actually testing truncation of lines - remove
def test_extreme_log_line_length_truncation():
"""This log has lines that are huge. Ensure we truncate the lines to 100"""
do_test("mozilla-central_ubuntu64_hw_test-androidx86-set-4-bm103-tests1-linux-build369")
@skip
def test_too_many_error_lines_truncation():
"""This log has a large number of lines that match the error regex. Ensure we truncate to 100 lines."""
do_test("large-number-of-error-lines")
@skip
def test_taskcluster_missing_finish_marker():
"""
A log from a Taskcluster job, where there was an infrastructure problem,

4
tests/log_parser/test_performance_artifact_builder.py
Просмотреть файл

@ -3,7 +3,7 @@ from jsonschema import validate
from tests.test_utils import add_log_response
from treeherder.log_parser.artifactbuildercollection import ArtifactBuilderCollection
from treeherder.log_parser.artifactbuilders import BuildbotPerformanceDataArtifactBuilder
from treeherder.log_parser.artifactbuilders import PerformanceDataArtifactBuilder
from treeherder.log_parser.utils import PERFHERDER_SCHEMA
@ -21,7 +21,7 @@ def test_performance_log_parsing():
]:
url = add_log_response(logfile)
builder = BuildbotPerformanceDataArtifactBuilder(url=url)
builder = PerformanceDataArtifactBuilder(url=url)
lpc = ArtifactBuilderCollection(url, builders=[builder])
lpc.parse()
act = lpc.artifacts[builder.name]

2
tests/model/test_classified_failure.py
Просмотреть файл

@ -81,7 +81,7 @@ def test_update_autoclassification_bug(test_job, test_job_2, classified_failures
mark_best_classification(text_log_errors[0], classified_failure)
assert classified_failure.bug_number is None
metadata = TextLogErrorMetadata.objects.get(text_log_error__step__job=test_job_2)
metadata = TextLogErrorMetadata.objects.get(text_log_error__job=test_job_2)
metadata.failure_line = FailureLine.objects.get(pk=3)
metadata.save()

157
tests/sample_data/artifacts/text_log_summary.json
Просмотреть файл

@ -1,125 +1,5 @@
{
"blob": {
"step_data": {
"steps": [
{
"errors": [],
"name": "set props: master",
"started": "2015-04-15 05:25:03.168328",
"started_linenumber": 8,
"finished_linenumber": 10,
"finished": "2015-04-15 05:25:03.168795",
"result": "success"
},
{
"errors": [],
"name": "set props: basedir",
"started": "2015-04-15 05:25:03.169099",
"started_linenumber": 12,
"finished_linenumber": 40,
"finished": "2015-04-15 05:25:03.597219",
"result": "success"
},
{
"errors": [],
"name": "downloading to buildprops.json",
"started": "2015-04-15 05:25:03.597542",
"started_linenumber": 42,
"finished_linenumber": 43,
"finished": "2015-04-15 05:25:03.726142",
"result": "success"
},
{
"errors": [],
"name": "'rm -rf ...'",
"started": "2015-04-15 05:25:03.726523",
"started_linenumber": 45,
"finished_linenumber": 71,
"finished": "2015-04-15 05:25:03.777120",
"result": "success"
},
{
"errors": [],
"name": "set props: script_repo_url",
"started": "2015-04-15 05:25:03.777445",
"started_linenumber": 73,
"finished_linenumber": 75,
"finished": "2015-04-15 05:25:03.777817",
"result": "success"
},
{
"errors": [],
"name": "'bash -c ...'",
"started": "2015-04-15 05:25:03.778747",
"started_linenumber": 77,
"finished_linenumber": 114,
"finished": "2015-04-15 05:25:03.945448",
"result": "success"
},
{
"errors": [],
"name": "set props: script_repo_revision script_repo_url",
"started": "2015-04-15 05:25:03.945809",
"started_linenumber": 116,
"finished_linenumber": 146,
"finished": "2015-04-15 05:25:04.682022",
"result": "success"
},
{
"errors": [],
"name": "'rm -rf ...'",
"started": "2015-04-15 05:25:04.682572",
"started_linenumber": 148,
"finished_linenumber": 174,
"finished": "2015-04-15 05:25:05.278125",
"result": "success"
},
{
"errors": [],
"name": "'hg clone ...'",
"started": "2015-04-15 05:25:05.278437",
"started_linenumber": 176,
"finished_linenumber": 210,
"finished": "2015-04-15 05:25:11.964370",
"result": "success"
},
{
"errors": [],
"name": "'hg update ...'",
"started": "2015-04-15 05:25:11.964712",
"started_linenumber": 212,
"finished_linenumber": 239,
"finished": "2015-04-15 05:25:12.506740",
"result": "success"
},
{
"errors": [],
"name": "set props: script_repo_revision",
"started": "2015-04-15 05:25:12.507514",
"started_linenumber": 241,
"finished_linenumber": 269,
"finished": "2015-04-15 05:25:12.700263",
"result": "success"
},
{
"errors": [],
"name": "downloading to oauth.txt",
"started": "2015-04-15 05:25:12.700568",
"started_linenumber": 271,
"finished_linenumber": 272,
"finished": "2015-04-15 05:25:12.730239",
"result": "success"
},
{
"errors": [],
"name": "tinderboxprint_script_revlink",
"started": "2015-04-15 05:25:12.730485",
"started_linenumber": 274,
"finished_linenumber": 276,
"finished": "2015-04-15 05:25:12.730832",
"result": "success"
},
{
"errors": [
{
"line": "05:35:49 INFO - 2018 INFO TEST-UNEXPECTED-FAIL | dom/indexedDB/test/test_transaction_lifetimes.html | Should be able to get objectStore - expected PASS",
@ -138,43 +18,6 @@
"linenumber": 8157
}
],
"name": "'/tools/buildbot/bin/python scripts/scripts/desktop_unittest.py ...' warnings",
"started": "2015-04-15 05:25:12.731202",
"started_linenumber": 278,
"finished_linenumber": 9248,
"finished": "2015-04-15 05:36:51.808779",
"result": "testfailed"
},
{
"errors": [],
"name": "set props: build_url blobber_files",
"started": "2015-04-15 05:36:51.812718",
"started_linenumber": 9250,
"finished_linenumber": 9280,
"finished": "2015-04-15 05:36:51.871786",
"result": "success"
},
{
"errors": [],
"name": "'rm -f ...'",
"started": "2015-04-15 05:36:51.872124",
"started_linenumber": 9282,
"finished_linenumber": 9308,
"finished": "2015-04-15 05:36:51.927004",
"result": "success"
},
{
"errors": [],
"name": "reboot skipped",
"started": "2015-04-15 05:36:51.927332",
"started_linenumber": 9310,
"finished_linenumber": 9311,
"finished": "2015-04-15 05:36:51.927750",
"result": "skipped"
}
],
"errors_truncated": false
},
"logurl": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/2015/04/2015-04-15-03-02-06-mozilla-central/mozilla-central_snowleopard_test-mochitest-2-bm107-tests1-macosx-build128.txt.gz"
},
"type": "json",

49
tests/sample_data/logs/crash-1.logview.json
Просмотреть файл

@ -1,25 +1,4 @@
{
"step_data": {
"steps": [
{
"errors": [],
"name": "set props: revision",
"started": "2013-06-05 12:51:48.767751",
"started_linenumber": 8,
"finished_linenumber": 10,
"finished": "2013-06-05 12:51:48.768393",
"result": "success"
},
{
"errors": [],
"name": "set props: who",
"started": "2013-06-05 12:51:48.768881",
"started_linenumber": 12,
"finished_linenumber": 14,
"finished": "2013-06-05 12:51:48.769473",
"result": "success"
},
{
"errors": [
{
"line": "Assertion failure: !(addr & GC_CELL_MASK), at e:/builds/moz2_slave/mozilla-central-win32-debug/build/js/src/jsgc.cpp:425",
@ -38,33 +17,5 @@
"linenumber": 39592
}
],
"name": "'python mochitest/runtests.py ...' warnings",
"started": "2013-06-05 12:54:55.232364",
"started_linenumber": 16,
"finished_linenumber": 39838,
"finished": "2013-06-05 13:15:39.235146",
"result": "testfailed"
},
{
"errors": [],
"name": "'rm -rf ...'",
"started": "2013-06-05 13:15:39.244545",
"started_linenumber": 39840,
"finished_linenumber": 39884,
"finished": "2013-06-05 13:16:04.014823",
"result": "success"
},
{
"errors": [],
"name": "maybe rebooting slave lost",
"started": "2013-06-05 13:16:04.015405",
"started_linenumber": 39886,
"finished_linenumber": 39888,
"finished": "2013-06-05 13:16:04.963614",
"result": "success"
}
],
"errors_truncated": false
},
"logurl": "http://my-log.mozilla.org/crash-1.txt.gz"
}

51
tests/sample_data/logs/jsreftest-fail.logview.json
Просмотреть файл

@ -1,29 +1,4 @@
{
"logurl": "http://my-log.mozilla.org/jsreftest-fail.txt.gz",
"step_data": {
"steps": [
{
"name": "set props: revision",
"started": "2013-06-05 12:51:48.767751",
"started_linenumber": 8,
"errors": [],
"finished": "2013-06-05 12:51:48.768393",
"finished_linenumber": 10,
"result": "success"
},
{
"name": "set props: who",
"started": "2013-06-05 12:51:48.768881",
"started_linenumber": 12,
"errors": [],
"finished": "2013-06-05 12:51:48.769473",
"finished_linenumber": 14,
"result": "success"
},
{
"name": "'python mochitest/runtests.py ...' warnings",
"started": "2013-06-05 12:54:55.232364",
"started_linenumber": 16,
"errors": [
{
"linenumber": 26994,
@ -234,29 +209,5 @@
"line": "580200: Assertion failure enumerating own properties of proxy returning duplicated own property name"
}
],
"finished": "2013-06-05 13:15:39.235146",
"finished_linenumber": 99347,
"result": "testfailed"
},
{
"name": "'rm -rf ...'",
"started": "2013-06-05 13:15:39.244545",
"started_linenumber": 99349,
"errors": [],
"finished": "2013-06-05 13:16:04.014823",
"finished_linenumber": 99393,
"result": "success"
},
{
"name": "maybe rebooting slave lost",
"started": "2013-06-05 13:16:04.015405",
"started_linenumber": 99395,
"errors": [],
"finished": "2013-06-05 13:16:04.963614",
"finished_linenumber": 99397,
"result": "success"
}
],
"errors_truncated": false
}
"logurl": "http://my-log.mozilla.org/jsreftest-fail.txt.gz"
}

503
tests/sample_data/logs/mozilla-central-win32-pgo-bm85-build1-build111.logview.json
Просмотреть файл

@ -1,503 +0,0 @@
{
"step_data": {
"steps": [
{
"errors": [],
"name": "set props: basedir",
"started": "2014-06-29 22:30:16.350054",
"started_linenumber": 8,
"finished_linenumber": 91,
"finished": "2014-06-29 22:30:17.733586",
"result": "success"
},
{
"errors": [],
"name": "set props: hashType",
"started": "2014-06-29 22:30:17.733997",
"started_linenumber": 93,
"finished_linenumber": 95,
"finished": "2014-06-29 22:30:17.734323",
"result": "success"
},
{
"errors": [],
"name": "set props: builddir",
"started": "2014-06-29 22:30:17.734607",
"started_linenumber": 97,
"finished_linenumber": 180,
"finished": "2014-06-29 22:30:17.852754",
"result": "success"
},
{
"errors": [],
"name": "clobber build tools",
"started": "2014-06-29 22:30:17.853076",
"started_linenumber": 182,
"finished_linenumber": 263,
"finished": "2014-06-29 22:30:29.974564",
"result": "success"
},
{
"errors": [],
"name": "clone build tools",
"started": "2014-06-29 22:30:29.975158",
"started_linenumber": 265,
"finished_linenumber": 353,
"finished": "2014-06-29 22:30:44.923107",
"result": "success"
},
{
"errors": [],
"name": "set props: toolsdir",
"started": "2014-06-29 22:30:44.923506",
"started_linenumber": 355,
"finished_linenumber": 438,
"finished": "2014-06-29 22:30:45.044162",
"result": "success"
},
{
"errors": [],
"name": "no change",
"started": "2014-06-29 22:30:45.045078",
"started_linenumber": 440,
"finished_linenumber": 522,
"finished": "2014-06-29 22:30:45.166864",
"result": "success"
},
{
"errors": [],
"name": "checking clobber times",
"started": "2014-06-29 22:30:45.167229",
"started_linenumber": 524,
"finished_linenumber": 712,
"finished": "2014-06-29 22:40:58.302111",
"result": "success"
},
{
"errors": [],
"name": "set props: purge_actual purge_target",
"started": "2014-06-29 22:40:58.302908",
"started_linenumber": 714,
"finished_linenumber": 811,
"finished": "2014-06-29 22:41:00.825003",
"result": "success"
},
{
"errors": [],
"name": "set_buildids",
"started": "2014-06-29 22:41:00.825666",
"started_linenumber": 813,
"finished_linenumber": 814,
"finished": "2014-06-29 22:41:00.825811",
"result": "success"
},
{
"errors": [],
"name": "set props: toolsdir",
"started": "2014-06-29 22:41:00.826350",
"started_linenumber": 816,
"finished_linenumber": 899,
"finished": "2014-06-29 22:41:00.948217",
"result": "success"
},
{
"errors": [],
"name": "set props: basedir",
"started": "2014-06-29 22:41:00.948537",
"started_linenumber": 901,
"finished_linenumber": 984,
"finished": "2014-06-29 22:41:01.066839",
"result": "success"
},
{
"errors": [],
"name": "remove mozharness",
"started": "2014-06-29 22:41:01.067220",
"started_linenumber": 986,
"finished_linenumber": 1067,
"finished": "2014-06-29 22:41:01.185636",
"result": "success"
},
{
"errors": [],
"name": "checkout mozharness",
"started": "2014-06-29 22:41:01.185958",
"started_linenumber": 1069,
"finished_linenumber": 1159,
"finished": "2014-06-29 22:41:06.592065",
"result": "success"
},
{
"errors": [],
"name": "updating mozharness to production",
"started": "2014-06-29 22:41:06.593516",
"started_linenumber": 1161,
"finished_linenumber": 1243,
"finished": "2014-06-29 22:41:07.116755",
"result": "success"
},
{
"errors": [],
"name": "delete old package",
"started": "2014-06-29 22:41:07.117128",
"started_linenumber": 1245,
"finished_linenumber": 1339,
"finished": "2014-06-29 22:41:07.635397",
"result": "success"
},
{
"errors": [],
"name": "downloading to buildprops.json",
"started": "2014-06-29 22:41:07.636162",
"started_linenumber": 1341,
"finished_linenumber": 1342,
"finished": "2014-06-29 22:41:07.757331",
"result": "success"
},
{
"errors": [],
"name": "'python c:/builds/moz2_slave/m-cen-w32-pgo-0000000000000000/tools/buildfarm/utils/retry.py ...'",
"started": "2014-06-29 22:41:07.757876",
"started_linenumber": 1344,
"finished_linenumber": 1517,
"finished": "2014-06-29 22:53:53.500864",
"result": "success"
},
{
"errors": [],
"name": "set props: got_revision",
"started": "2014-06-29 22:53:53.503996",
"started_linenumber": 1519,
"finished_linenumber": 1601,
"finished": "2014-06-29 22:53:53.723762",
"result": "success"
},
{
"errors": [],
"name": "set props: comments",
"started": "2014-06-29 22:53:53.724120",
"started_linenumber": 1603,
"finished_linenumber": 1605,
"finished": "2014-06-29 22:53:53.724565",
"result": "success"
},
{
"errors": [],
"name": "got mozconfig",
"started": "2014-06-29 22:53:53.724904",
"started_linenumber": 1607,
"finished_linenumber": 1691,
"finished": "2014-06-29 22:53:54.184027",
"result": "success"
},
{
"errors": [],
"name": "'cat .mozconfig'",
"started": "2014-06-29 22:53:54.184825",
"started_linenumber": 1693,
"finished_linenumber": 1785,
"finished": "2014-06-29 22:53:54.304847",
"result": "success"
},
{
"errors": [],
"name": "'sh c:/builds/moz2_slave/m-cen-w32-pgo-0000000000000000/tools/scripts/tooltool/tooltool_wrapper.sh ...'",
"started": "2014-06-29 22:53:54.305271",
"started_linenumber": 1787,
"finished_linenumber": 1910,
"finished": "2014-06-29 22:53:56.138113",
"result": "success"
},
{
"errors": [],
"name": "remove old nonce",
"started": "2014-06-29 22:53:56.138958",
"started_linenumber": 1912,
"finished_linenumber": 1993,
"finished": "2014-06-29 22:53:56.257351",
"result": "success"
},
{
"errors": [],
"name": "downloading to token",
"started": "2014-06-29 22:53:56.257721",
"started_linenumber": 1995,
"finished_linenumber": 2000,
"finished": "2014-06-29 22:53:56.469662",
"result": "success"
},
{
"errors": [],
"name": "compile",
"started": "2014-06-29 22:53:56.470289",
"started_linenumber": 2002,
"finished_linenumber": 57536,
"finished": "2014-06-30 02:21:57.813004",
"result": "success"
},
{
"errors": [],
"name": "set props: vsize testresults",
"started": "2014-06-30 02:21:57.815584",
"started_linenumber": 57538,
"finished_linenumber": 57622,
"finished": "2014-06-30 02:21:57.958564",
"result": "success"
},
{
"errors": [],
"name": "set props: buildid",
"started": "2014-06-30 02:21:57.958855",
"started_linenumber": 57624,
"finished_linenumber": 57707,
"finished": "2014-06-30 02:22:00.177483",
"result": "success"
},
{
"errors": [],
"name": "set props: sourcestamp",
"started": "2014-06-30 02:22:00.177859",
"started_linenumber": 57709,
"finished_linenumber": 57792,
"finished": "2014-06-30 02:22:00.298879",
"result": "success"
},
{
"errors": [],
"name": "downloading to properties.json",
"started": "2014-06-30 02:22:00.299227",
"started_linenumber": 57794,
"finished_linenumber": 57795,
"finished": "2014-06-30 02:22:00.313580",
"result": "success"
},
{
"errors": [],
"name": "graph server post results complete",
"started": "2014-06-30 02:22:00.314328",
"started_linenumber": 57797,
"finished_linenumber": 57896,
"finished": "2014-06-30 02:22:01.356388",
"result": "success"
},
{
"errors": [],
"name": "'python c:/builds/moz2_slave/m-cen-w32-pgo-0000000000000000/build/build/pymake/make.py ...'",
"started": "2014-06-30 02:22:01.357389",
"started_linenumber": 57898,
"finished_linenumber": 58630,
"finished": "2014-06-30 02:27:10.574663",
"result": "success"
},
{
"errors": [],
"name": "'python c:/builds/moz2_slave/m-cen-w32-pgo-0000000000000000/build/build/pymake/make.py ...'",
"started": "2014-06-30 02:27:10.575695",
"started_linenumber": 58632,
"finished_linenumber": 59131,
"finished": "2014-06-30 02:34:37.112183",
"result": "success"
},
{
"errors": [],
"name": "'python c:/builds/moz2_slave/m-cen-w32-pgo-0000000000000000/build/build/pymake/make.py ...'",
"started": "2014-06-30 02:34:37.113149",
"started_linenumber": 59133,
"finished_linenumber": 63272,
"finished": "2014-06-30 02:36:25.733582",
"result": "success"
},
{
"errors": [],
"name": "find filepath",
"started": "2014-06-30 02:36:25.734713",
"started_linenumber": 63274,
"finished_linenumber": 63356,
"finished": "2014-06-30 02:36:50.759546",
"result": "success"
},
{
"errors": [],
"name": "set props: packageFilename",
"started": "2014-06-30 02:36:50.759940",
"started_linenumber": 63358,
"finished_linenumber": 63441,
"finished": "2014-06-30 02:36:50.883981",
"result": "success"
},
{
"errors": [],
"name": "set props: packageSize",
"started": "2014-06-30 02:36:50.884836",
"started_linenumber": 63443,
"finished_linenumber": 63526,
"finished": "2014-06-30 02:36:51.003225",
"result": "success"
},
{
"errors": [],
"name": "set props: packageHash",
"started": "2014-06-30 02:36:51.003640",
"started_linenumber": 63528,
"finished_linenumber": 63611,
"finished": "2014-06-30 02:36:51.727337",
"result": "success"
},
{
"errors": [],
"name": "set props: filepath",
"started": "2014-06-30 02:36:51.728149",
"started_linenumber": 63613,
"finished_linenumber": 63696,
"finished": "2014-06-30 02:36:51.852435",
"result": "success"
},
{
"errors": [],
"name": "'python c:/builds/moz2_slave/m-cen-w32-pgo-0000000000000000/build/build/pymake/make.py ...'",
"started": "2014-06-30 02:36:51.853240",
"started_linenumber": 63698,
"finished_linenumber": 67508,
"finished": "2014-06-30 02:38:19.792708",
"result": "success"
},
{
"errors": [],
"name": "find filepath",
"started": "2014-06-30 02:38:19.797776",
"started_linenumber": 67510,
"finished_linenumber": 67592,
"finished": "2014-06-30 02:38:20.178990",
"result": "success"
},
{
"errors": [],
"name": "set props: installerFilename",
"started": "2014-06-30 02:38:20.179426",
"started_linenumber": 67594,
"finished_linenumber": 67677,
"finished": "2014-06-30 02:38:20.299869",
"result": "success"
},
{
"errors": [],
"name": "set props: installerSize",
"started": "2014-06-30 02:38:20.300615",
"started_linenumber": 67679,
"finished_linenumber": 67762,
"finished": "2014-06-30 02:38:20.422898",
"result": "success"
},
{
"errors": [],
"name": "set props: installerHash",
"started": "2014-06-30 02:38:20.423321",
"started_linenumber": 67764,
"finished_linenumber": 67847,
"finished": "2014-06-30 02:38:20.746602",
"result": "success"
},
{
"errors": [],
"name": "set props: filepath",
"started": "2014-06-30 02:38:20.747658",
"started_linenumber": 67849,
"finished_linenumber": 67932,
"finished": "2014-06-30 02:38:20.866610",
"result": "success"
},
{
"errors": [],
"name": "set props: buildid",
"started": "2014-06-30 02:38:20.867042",
"started_linenumber": 67934,
"finished_linenumber": 68017,
"finished": "2014-06-30 02:38:20.988577",
"result": "success"
},
{
"errors": [],
"name": "set props: appVersion",
"started": "2014-06-30 02:38:20.989188",
"started_linenumber": 68019,
"finished_linenumber": 68102,
"finished": "2014-06-30 02:38:21.111717",
"result": "success"
},
{
"errors": [],
"name": "set props: appName",
"started": "2014-06-30 02:38:21.112102",
"started_linenumber": 68104,
"finished_linenumber": 68187,
"finished": "2014-06-30 02:38:21.235722",
"result": "success"
},
{
"errors": [],
"name": "set props: symbolsUrl packageUrl testsUrl jsshellUrl",
"started": "2014-06-30 02:38:21.236747",
"started_linenumber": 68189,
"finished_linenumber": 68366,
"finished": "2014-06-30 02:38:44.118836",
"result": "success"
},
{
"errors": [],
"name": "set build property skipped",
"started": "2014-06-30 02:38:44.120204",
"started_linenumber": 68368,
"finished_linenumber": 68369,
"finished": "2014-06-30 02:38:44.120762",
"result": "skipped"
},
{
"errors": [],
"name": "sendchange",
"started": "2014-06-30 02:38:44.121114",
"started_linenumber": 68371,
"finished_linenumber": 68480,
"finished": "2014-06-30 02:38:51.041903",
"result": "success"
},
{
"errors": [],
"name": "sendchange",
"started": "2014-06-30 02:38:51.042816",
"started_linenumber": 68482,
"finished_linenumber": 68591,
"finished": "2014-06-30 02:38:51.425600",
"result": "success"
},
{
"errors": [],
"name": "'python c:/builds/moz2_slave/m-cen-w32-pgo-0000000000000000/build/build/pymake/make.py ...'",
"started": "2014-06-30 02:38:51.426344",
"started_linenumber": 68593,
"finished_linenumber": 70179,
"finished": "2014-06-30 02:40:35.920512",
"result": "success"
},
{
"errors": [],
"name": "check test complete",
"started": "2014-06-30 02:40:35.921976",
"started_linenumber": 70181,
"finished_linenumber": 80063,
"finished": "2014-06-30 02:45:56.094867",
"result": "success"
},
{
"errors": [],
"name": "maybe rebooting slave lost",
"started": "2014-06-30 02:45:56.099903",
"started_linenumber": 80065,
"finished_linenumber": 80067,
"finished": "2014-06-30 02:45:56.743970",
"result": "success"
}
],
"errors_truncated": false
},
"logurl": "http://my-log.mozilla.org/mozilla-central-win32-pgo-bm85-build1-build111.txt.gz"
}

16
tests/sample_data/logs/mozilla-central_fedora-b2g_test-crashtest-1-bm54-tests1-linux-build50.jobartifact.json
Просмотреть файл

@ -1,16 +0,0 @@
{
"job_details": [
{
"url": "http://hg.mozilla.org/build/mozharness/rev/c43ba6cb3db3",
"content_type": "link",
"value": "http://hg.mozilla.org/build/mozharness/rev/c43ba6cb3db3",
"title": "mozharness_revlink"
},
{
"content_type": "raw_html",
"value": "810/0/22",
"title": "crashtest"
}
],
"logurl": "http://my-log.mozilla.org/mozilla-central_fedora-b2g_test-crashtest-1-bm54-tests1-linux-build50.txt.gz"
}

116
tests/sample_data/logs/mozilla-central_fedora-b2g_test-crashtest-1-bm54-tests1-linux-build50.logview.json
Просмотреть файл

@ -1,116 +0,0 @@
{
"step_data": {
"steps": [
{
"errors": [],
"name": "set props: master",
"started": "2013-06-05 12:39:57.838527",
"started_linenumber": 8,
"finished_linenumber": 10,
"finished": "2013-06-05 12:39:57.839226",
"result": "success"
},
{
"errors": [],
"name": "set props: basedir",
"started": "2013-06-05 12:39:57.839702",
"started_linenumber": 12,
"finished_linenumber": 42,
"finished": "2013-06-05 12:39:58.346114",
"result": "success"
},
{
"errors": [],
"name": "downloading to buildprops.json",
"started": "2013-06-05 12:39:58.346573",
"started_linenumber": 44,
"finished_linenumber": 45,
"finished": "2013-06-05 12:39:58.614586",
"result": "success"
},
{
"errors": [],
"name": "'rm -rf ...'",
"started": "2013-06-05 12:39:58.615003",
"started_linenumber": 47,
"finished_linenumber": 75,
"finished": "2013-06-05 12:39:58.723069",
"result": "success"
},
{
"errors": [],
"name": "'rm -rf ...'",
"started": "2013-06-05 12:39:58.723521",
"started_linenumber": 77,
"finished_linenumber": 105,
"finished": "2013-06-05 12:39:58.831972",
"result": "success"
},
{
"errors": [],
"name": "'hg clone ...'",
"started": "2013-06-05 12:39:58.832419",
"started_linenumber": 107,
"finished_linenumber": 142,
"finished": "2013-06-05 12:40:01.114413",
"result": "success"
},
{
"errors": [],
"name": "'hg update ...'",
"started": "2013-06-05 12:40:01.114870",
"started_linenumber": 144,
"finished_linenumber": 173,
"finished": "2013-06-05 12:40:01.335999",
"result": "success"
},
{
"errors": [],
"name": "set props: script_repo_revision",
"started": "2013-06-05 12:40:01.336478",
"started_linenumber": 175,
"finished_linenumber": 205,
"finished": "2013-06-05 12:40:01.520234",
"result": "success"
},
{
"errors": [],
"name": "tinderboxprint_script_revlink",
"started": "2013-06-05 12:40:01.520734",
"started_linenumber": 207,
"finished_linenumber": 209,
"finished": "2013-06-05 12:40:01.521320",
"result": "success"
},
{
"errors": [],
"name": "'/tools/buildbot/bin/python scripts/scripts/b2g_emulator_unittest.py ...'",
"started": "2013-06-05 12:40:01.521797",
"started_linenumber": 211,
"finished_linenumber": 3369,
"finished": "2013-06-05 12:57:55.646752",
"result": "success"
},
{
"errors": [],
"name": "set props: build_url",
"started": "2013-06-05 12:57:55.659998",
"started_linenumber": 3371,
"finished_linenumber": 3401,
"finished": "2013-06-05 12:57:55.793402",
"result": "success"
},
{
"errors": [],
"name": "reboot slave lost",
"started": "2013-06-05 12:57:55.793914",
"started_linenumber": 3403,
"finished_linenumber": 3433,
"finished": "2013-06-05 12:58:55.994641",
"result": "success"
}
],
"errors_truncated": false
},
"logurl": "http://my-log.mozilla.org/mozilla-central_fedora-b2g_test-crashtest-1-bm54-tests1-linux-build50.txt.gz"
}

16
tests/sample_data/logs/mozilla-central_mountainlion_test-mochitest-2-bm77-tests1-macosx-build141.jobartifact.json
Просмотреть файл

@ -1,16 +0,0 @@
{
"job_details": [
{
"url": "http://hg.mozilla.org/build/mozharness/rev/c43ba6cb3db3",
"content_type": "link",
"value": "http://hg.mozilla.org/build/mozharness/rev/c43ba6cb3db3",
"title": "mozharness_revlink"
},
{
"content_type": "raw_html",
"value": "206415/0/18500",
"title": "mochitest-plain2"
}
],
"logurl": "http://my-log.mozilla.org/mozilla-central_mountainlion_test-mochitest-2-bm77-tests1-macosx-build141.txt.gz"
}

116
tests/sample_data/logs/mozilla-central_mountainlion_test-mochitest-2-bm77-tests1-macosx-build141.logview.json
Просмотреть файл

@ -1,116 +0,0 @@
{
"step_data": {
"steps": [
{
"errors": [],
"name": "set props: master",
"started": "2013-06-05 12:55:02.793687",
"started_linenumber": 8,
"finished_linenumber": 10,
"finished": "2013-06-05 12:55:02.794485",
"result": "success"
},
{
"errors": [],
"name": "set props: basedir",
"started": "2013-06-05 12:55:02.794985",
"started_linenumber": 12,
"finished_linenumber": 36,
"finished": "2013-06-05 12:55:03.579905",
"result": "success"
},
{
"errors": [],
"name": "downloading to buildprops.json",
"started": "2013-06-05 12:55:03.580420",
"started_linenumber": 38,
"finished_linenumber": 39,
"finished": "2013-06-05 12:55:05.429410",
"result": "success"
},
{
"errors": [],
"name": "'rm -rf ...'",
"started": "2013-06-05 12:55:05.429870",
"started_linenumber": 41,
"finished_linenumber": 63,
"finished": "2013-06-05 12:55:05.711713",
"result": "success"
},
{
"errors": [],
"name": "'rm -rf ...'",
"started": "2013-06-05 12:55:05.712188",
"started_linenumber": 65,
"finished_linenumber": 87,
"finished": "2013-06-05 12:55:05.951055",
"result": "success"
},
{
"errors": [],
"name": "'hg clone ...'",
"started": "2013-06-05 12:55:05.951563",
"started_linenumber": 89,
"finished_linenumber": 118,
"finished": "2013-06-05 12:55:08.587665",
"result": "success"
},
{
"errors": [],
"name": "'hg update ...'",
"started": "2013-06-05 12:55:08.588162",
"started_linenumber": 120,
"finished_linenumber": 143,
"finished": "2013-06-05 12:55:09.070938",
"result": "success"
},
{
"errors": [],
"name": "set props: script_repo_revision",
"started": "2013-06-05 12:55:09.071436",
"started_linenumber": 145,
"finished_linenumber": 169,
"finished": "2013-06-05 12:55:09.553968",
"result": "success"
},
{
"errors": [],
"name": "tinderboxprint_script_revlink",
"started": "2013-06-05 12:55:09.554462",
"started_linenumber": 171,
"finished_linenumber": 173,
"finished": "2013-06-05 12:55:09.555053",
"result": "success"
},
{
"errors": [],
"name": "'/tools/buildbot/bin/python scripts/scripts/desktop_unittest.py ...'",
"started": "2013-06-05 12:55:09.555542",
"started_linenumber": 175,
"finished_linenumber": 11552,
"finished": "2013-06-05 13:02:38.681825",
"result": "success"
},
{
"errors": [],
"name": "set props: build_url",
"started": "2013-06-05 13:02:38.687346",
"started_linenumber": 11554,
"finished_linenumber": 11578,
"finished": "2013-06-05 13:02:38.876303",
"result": "success"
},
{
"errors": [],
"name": "reboot slave lost",
"started": "2013-06-05 13:02:38.876850",
"started_linenumber": 11580,
"finished_linenumber": 11609,
"finished": "2013-06-05 13:03:00.115654",
"result": "success"
}
],
"errors_truncated": false
},
"logurl": "http://my-log.mozilla.org/mozilla-central_mountainlion_test-mochitest-2-bm77-tests1-macosx-build141.txt.gz"
}

130
tests/sample_data/logs/mozilla-central_ubuntu64_hw_test-androidx86-set-4-bm103-tests1-linux-build369.logview.json
Просмотреть файл

@ -1,97 +1,4 @@
{
"step_data": {
"steps": [
{
"errors": [],
"name": "set props: master",
"started": "2014-07-17 08:45:18.801609",
"started_linenumber": 8,
"finished_linenumber": 10,
"finished": "2014-07-17 08:45:18.802011",
"result": "success"
},
{
"errors": [],
"name": "set props: basedir",
"started": "2014-07-17 08:45:18.802295",
"started_linenumber": 12,
"finished_linenumber": 56,
"finished": "2014-07-17 08:45:18.854874",
"result": "success"
},
{
"errors": [],
"name": "downloading to buildprops.json",
"started": "2014-07-17 08:45:18.855272",
"started_linenumber": 58,
"finished_linenumber": 59,
"finished": "2014-07-17 08:45:18.904210",
"result": "success"
},
{
"errors": [],
"name": "'rm -rf ...'",
"started": "2014-07-17 08:45:18.904577",
"started_linenumber": 61,
"finished_linenumber": 103,
"finished": "2014-07-17 08:45:18.943479",
"result": "success"
},
{
"errors": [],
"name": "'rm -rf ...'",
"started": "2014-07-17 08:45:18.943846",
"started_linenumber": 105,
"finished_linenumber": 147,
"finished": "2014-07-17 08:45:19.043322",
"result": "success"
},
{
"errors": [],
"name": "'hg clone ...'",
"started": "2014-07-17 08:45:19.043748",
"started_linenumber": 149,
"finished_linenumber": 198,
"finished": "2014-07-17 08:45:21.830887",
"result": "success"
},
{
"errors": [],
"name": "'hg update ...'",
"started": "2014-07-17 08:45:21.831343",
"started_linenumber": 200,
"finished_linenumber": 243,
"finished": "2014-07-17 08:45:22.037742",
"result": "success"
},
{
"errors": [],
"name": "set props: script_repo_revision",
"started": "2014-07-17 08:45:22.038178",
"started_linenumber": 245,
"finished_linenumber": 289,
"finished": "2014-07-17 08:45:22.126650",
"result": "success"
},
{
"errors": [],
"name": "downloading to oauth.txt",
"started": "2014-07-17 08:45:22.126969",
"started_linenumber": 291,
"finished_linenumber": 292,
"finished": "2014-07-17 08:45:22.140342",
"result": "success"
},
{
"errors": [],
"name": "tinderboxprint_script_revlink",
"started": "2014-07-17 08:45:22.140752",
"started_linenumber": 294,
"finished_linenumber": 296,
"finished": "2014-07-17 08:45:22.141210",
"result": "success"
},
{
"errors": [
{
"line": "09:14:00 INFO - org.json.JSONException: Unterminated string at character 23963 of {\"reason\":\"android-anr-report\",\"slug\":\"5e99c087-ebb5-4091-b5c8-75bd0889509c\",\"payload\":{\"ver\":1,\"simpleMeasurements\":{\"uptime\":0},\"info\":{\"reason\":\"android-anr-report\",\"OS\":\"Android\",\"version\":\"17\",\"appID\":\"{aa3c5121-dab2-40e2-81ca-7ea25febc110}\",\"appVersion\":\"33.0a1\",\"appName\":\"Fennec\",\"appBuildID\":\"20140717073317\",\"appUpdateChannel\":\"default\",\"platformBuildID\":\"20140717073317\",\"locale\":\"en-US\",\"cpucount\":1,\"",
@ -102,42 +9,5 @@
"linenumber": 3571
}
],
"name": "'/tools/buildbot/bin/python scripts/scripts/android_emulator_unittest.py ...' warnings",
"started": "2014-07-17 08:45:22.141524",
"started_linenumber": 298,
"finished_linenumber": 10231,
"finished": "2014-07-17 09:18:15.066397",
"result": "testfailed"
},
{
"errors": [],
"name": "set props: blobber_manifest_url build_url",
"started": "2014-07-17 09:18:15.071084",
"started_linenumber": 10233,
"finished_linenumber": 10279,
"finished": "2014-07-17 09:18:15.266212",
"result": "success"
},
{
"errors": [],
"name": "'rm -f ...'",
"started": "2014-07-17 09:18:15.266604",
"started_linenumber": 10281,
"finished_linenumber": 10323,
"finished": "2014-07-17 09:18:15.290870",
"result": "success"
},
{
"errors": [],
"name": "reboot slave lost",
"started": "2014-07-17 09:18:15.291165",
"started_linenumber": 10325,
"finished_linenumber": 10369,
"finished": "2014-07-17 09:18:22.104200",
"result": "success"
}
],
"errors_truncated": false
},
"logurl": "http://my-log.mozilla.org/mozilla-central_ubuntu64_hw_test-androidx86-set-4-bm103-tests1-linux-build369.txt.gz"
}

211
tests/sample_data/logs/mozilla-esr17_xp_test_pgo-mochitest-browser-chrome-bm74-tests1-windows-build12.logview.json
Просмотреть файл

@ -1,187 +1,4 @@
{
"step_data": {
"steps": [
{
"errors": [],
"name": "set props: revision",
"started": "2013-06-05 12:51:48.767751",
"started_linenumber": 8,
"finished_linenumber": 10,
"finished": "2013-06-05 12:51:48.768393",
"result": "success"
},
{
"errors": [],
"name": "set props: who",
"started": "2013-06-05 12:51:48.768881",
"started_linenumber": 12,
"finished_linenumber": 14,
"finished": "2013-06-05 12:51:48.769473",
"result": "success"
},
{
"errors": [],
"name": "set props: basedir",
"started": "2013-06-05 12:51:48.769970",
"started_linenumber": 16,
"finished_linenumber": 62,
"finished": "2013-06-05 12:51:50.898723",
"result": "success"
},
{
"errors": [],
"name": "set props: hashType",
"started": "2013-06-05 12:51:50.899244",
"started_linenumber": 64,
"finished_linenumber": 110,
"finished": "2013-06-05 12:51:51.050525",
"result": "success"
},
{
"errors": [],
"name": "set props: builddir",
"started": "2013-06-05 12:51:51.051012",
"started_linenumber": 112,
"finished_linenumber": 158,
"finished": "2013-06-05 12:51:51.285053",
"result": "success"
},
{
"errors": [],
"name": "clobber build tools",
"started": "2013-06-05 12:51:51.285559",
"started_linenumber": 160,
"finished_linenumber": 204,
"finished": "2013-06-05 12:52:04.344452",
"result": "success"
},
{
"errors": [],
"name": "clone build tools",
"started": "2013-06-05 12:52:04.344949",
"started_linenumber": 206,
"finished_linenumber": 257,
"finished": "2013-06-05 12:52:27.906228",
"result": "success"
},
{
"errors": [],
"name": "set props: toolsdir",
"started": "2013-06-05 12:52:27.906738",
"started_linenumber": 259,
"finished_linenumber": 305,
"finished": "2013-06-05 12:52:28.096555",
"result": "success"
},
{
"errors": [],
"name": "set props: purge_actual purge_target",
"started": "2013-06-05 12:52:28.097052",
"started_linenumber": 307,
"finished_linenumber": 373,
"finished": "2013-06-05 12:52:28.689661",
"result": "success"
},
{
"errors": [],
"name": "'rm -rf ...'",
"started": "2013-06-05 12:52:28.690156",
"started_linenumber": 375,
"finished_linenumber": 419,
"finished": "2013-06-05 12:53:56.014058",
"result": "success"
},
{
"errors": [],
"name": "download",
"started": "2013-06-05 12:53:56.014651",
"started_linenumber": 421,
"finished_linenumber": 484,
"finished": "2013-06-05 12:54:02.983974",
"result": "success"
},
{
"errors": [],
"name": "unpack",
"started": "2013-06-05 12:54:02.984470",
"started_linenumber": 486,
"finished_linenumber": 530,
"finished": "2013-06-05 12:54:06.858348",
"result": "success"
},
{
"errors": [],
"name": "set props: exepath",
"started": "2013-06-05 12:54:06.858854",
"started_linenumber": 532,
"finished_linenumber": 534,
"finished": "2013-06-05 12:54:06.859521",
"result": "success"
},
{
"errors": [],
"name": "set props: exedir",
"started": "2013-06-05 12:54:06.859991",
"started_linenumber": 536,
"finished_linenumber": 538,
"finished": "2013-06-05 12:54:06.860629",
"result": "success"
},
{
"errors": [],
"name": "set props: toolsdir",
"started": "2013-06-05 12:54:06.861112",
"started_linenumber": 540,
"finished_linenumber": 586,
"finished": "2013-06-05 12:54:07.047366",
"result": "success"
},
{
"errors": [],
"name": "set props: symbols_url",
"started": "2013-06-05 12:54:07.047887",
"started_linenumber": 588,
"finished_linenumber": 590,
"finished": "2013-06-05 12:54:07.048555",
"result": "success"
},
{
"errors": [],
"name": "download",
"started": "2013-06-05 12:54:07.049041",
"started_linenumber": 592,
"finished_linenumber": 665,
"finished": "2013-06-05 12:54:22.389733",
"result": "success"
},
{
"errors": [],
"name": "set props: buildid",
"started": "2013-06-05 12:54:22.390374",
"started_linenumber": 667,
"finished_linenumber": 714,
"finished": "2013-06-05 12:54:22.547237",
"result": "success"
},
{
"errors": [],
"name": "'python.exe c:/talos-slave/test/tools/scripts/support/mouse_and_screen_resolution.py ...'",
"started": "2013-06-05 12:54:22.547836",
"started_linenumber": 716,
"finished_linenumber": 761,
"finished": "2013-06-05 12:54:23.359326",
"result": "success"
},
{
"errors": [],
"name": "unpack tests",
"started": "2013-06-05 12:54:23.359896",
"started_linenumber": 763,
"finished_linenumber": 807,
"finished": "2013-06-05 12:54:55.231690",
"result": "success"
},
{
"errors": [
{
"line": "TEST-UNEXPECTED-FAIL | chrome://mochitests/content/browser/dom/tests/browser/browser_ConsoleAPITests.js | Exception thrown in CO_observe: TypeError: Components.utils.isXrayWrapper is not a function",
@ -192,33 +9,5 @@
"linenumber": 26752
}
],
"name": "'python mochitest/runtests.py ...' warnings",
"started": "2013-06-05 12:54:55.232364",
"started_linenumber": 809,
"finished_linenumber": 54531,
"finished": "2013-06-05 13:15:39.235146",
"result": "testfailed"
},
{
"errors": [],
"name": "'rm -rf ...'",
"started": "2013-06-05 13:15:39.244545",
"started_linenumber": 54533,
"finished_linenumber": 54577,
"finished": "2013-06-05 13:16:04.014823",
"result": "success"
},
{
"errors": [],
"name": "maybe rebooting slave lost",
"started": "2013-06-05 13:16:04.015405",
"started_linenumber": 54579,
"finished_linenumber": 54581,
"finished": "2013-06-05 13:16:04.963614",
"result": "success"
}
],
"errors_truncated": false
},
"logurl": "http://my-log.mozilla.org/mozilla-esr17_xp_test_pgo-mochitest-browser-chrome-bm74-tests1-windows-build12.txt.gz"
}

112
tests/sample_data/logs/mozilla-inbound_ubuntu64_vm-debug_test-mochitest-other-bm53-tests1-linux-build122.logview.json
Просмотреть файл

@ -1,88 +1,4 @@
{
"step_data": {
"steps": [
{
"errors": [],
"name": "set props: master",
"started": "2013-06-05 12:44:58.842738",
"started_linenumber": 8,
"finished_linenumber": 10,
"finished": "2013-06-05 12:44:58.843804",
"result": "success"
},
{
"errors": [],
"name": "set props: basedir",
"started": "2013-06-05 12:44:58.844579",
"started_linenumber": 12,
"finished_linenumber": 52,
"finished": "2013-06-05 12:44:58.975106",
"result": "success"
},
{
"errors": [],
"name": "downloading to buildprops.json",
"started": "2013-06-05 12:44:58.975618",
"started_linenumber": 54,
"finished_linenumber": 55,
"finished": "2013-06-05 12:44:59.037644",
"result": "success"
},
{
"errors": [],
"name": "'rm -rf ...'",
"started": "2013-06-05 12:44:59.038141",
"started_linenumber": 57,
"finished_linenumber": 95,
"finished": "2013-06-05 12:44:59.112857",
"result": "success"
},
{
"errors": [],
"name": "'rm -rf ...'",
"started": "2013-06-05 12:44:59.113592",
"started_linenumber": 97,
"finished_linenumber": 135,
"finished": "2013-06-05 12:44:59.201908",
"result": "success"
},
{
"errors": [],
"name": "'hg clone ...'",
"started": "2013-06-05 12:44:59.202642",
"started_linenumber": 137,
"finished_linenumber": 182,
"finished": "2013-06-05 12:45:02.180652",
"result": "success"
},
{
"errors": [],
"name": "'hg update ...'",
"started": "2013-06-05 12:45:02.181421",
"started_linenumber": 184,
"finished_linenumber": 223,
"finished": "2013-06-05 12:45:02.421021",
"result": "success"
},
{
"errors": [],
"name": "set props: script_repo_revision",
"started": "2013-06-05 12:45:02.421566",
"started_linenumber": 225,
"finished_linenumber": 265,
"finished": "2013-06-05 12:45:02.558016",
"result": "success"
},
{
"errors": [],
"name": "tinderboxprint_script_revlink",
"started": "2013-06-05 12:45:02.558553",
"started_linenumber": 267,
"finished_linenumber": 269,
"finished": "2013-06-05 12:45:02.559173",
"result": "success"
},
{
"errors": [
{
"line": "12:46:31 INFO - Assertion failure: pNew->iVersion == pSub->iVersion, at ../../../storage/src/TelemetryVFS.cpp:357",
@ -133,33 +49,5 @@
"linenumber": 3062
}
],
"name": "'/tools/buildbot/bin/python scripts/scripts/desktop_unittest.py ...' warnings",
"started": "2013-06-05 12:45:02.559674",
"started_linenumber": 271,
"finished_linenumber": 3070,
"finished": "2013-06-05 12:46:55.430095",
"result": "testfailed"
},
{
"errors": [],
"name": "set props: build_url symbols_url",
"started": "2013-06-05 12:46:55.431212",
"started_linenumber": 3072,
"finished_linenumber": 3114,
"finished": "2013-06-05 12:46:55.479840",
"result": "success"
},
{
"errors": [],
"name": "reboot slave lost",
"started": "2013-06-05 12:46:55.480585",
"started_linenumber": 3116,
"finished_linenumber": 3118,
"finished": "2013-06-05 12:46:57.867188",
"result": "success"
}
],
"errors_truncated": false
},
"logurl": "http://my-log.mozilla.org/mozilla-inbound_ubuntu64_vm-debug_test-mochitest-other-bm53-tests1-linux-build122.txt.gz"
}

51
tests/sample_data/logs/opt-objc-exception.logview.json
Просмотреть файл

@ -1,29 +1,4 @@
{
"logurl": "http://my-log.mozilla.org/opt-objc-exception.txt.gz",
"step_data": {
"steps": [
{
"name": "set props: revision",
"started": "2013-06-05 12:51:48.767751",
"started_linenumber": 8,
"errors": [],
"finished": "2013-06-05 12:51:48.768393",
"finished_linenumber": 10,
"result": "success"
},
{
"name": "set props: who",
"started": "2013-06-05 12:51:48.768881",
"started_linenumber": 12,
"errors": [],
"finished": "2013-06-05 12:51:48.769473",
"finished_linenumber": 14,
"result": "success"
},
{
"name": "'python mochitest/runtests.py ...' warnings",
"started": "2013-06-05 12:54:55.232364",
"started_linenumber": 16,
"errors": [
{
"linenumber": 24056,
@ -38,29 +13,5 @@
"line": "2010-09-23 14:48:01.299 firefox-bin[319:10b] *** Assertion failure in -[NSNextStepFrame lockFocus], /SourceCache/AppKit/AppKit-949.54/AppKit.subproj/NSView.m:4755"
}
],
"finished": "2013-06-05 13:15:39.235146",
"finished_linenumber": 48792,
"result": "testfailed"
},
{
"name": "'rm -rf ...'",
"started": "2013-06-05 13:15:39.244545",
"started_linenumber": 48794,
"errors": [],
"finished": "2013-06-05 13:16:04.014823",
"finished_linenumber": 48838,
"result": "success"
},
{
"name": "maybe rebooting slave lost",
"started": "2013-06-05 13:16:04.015405",
"started_linenumber": 48840,
"errors": [],
"finished": "2013-06-05 13:16:04.963614",
"finished_linenumber": 48842,
"result": "success"
}
],
"errors_truncated": false
}
"logurl": "http://my-log.mozilla.org/opt-objc-exception.txt.gz"
}

2
tests/ui/job-view/Filtering_test.jsx
Просмотреть файл

@ -186,7 +186,7 @@ describe('Filtering', () => {
[],
);
fetchMock.get(
getProjectUrl('/jobs/259537372/text_log_steps/', 'autoland'),
getProjectUrl('/jobs/259537372/text_log_errors/', 'autoland'),
[],
);
});

2
tests/ui/job-view/details/PinBoard_test.jsx
Просмотреть файл

@ -41,7 +41,7 @@ describe('DetailsPanel', () => {
fetchMock.get(getProjectUrl('/jobs/259537372/', repoName), jobList.data[1]);
fetchMock.get(getProjectUrl('/note/?job_id=259537372', repoName), []);
fetchMock.get(
getProjectUrl('/jobs/259537372/text_log_steps/', repoName),
getProjectUrl('/jobs/259537372/text_log_errors/', repoName),
[],
);
fetchMock.get(

2
tests/ui/mock/full_job.json
Просмотреть файл

@ -36,7 +36,7 @@
"resource_uri": "/api/project/try/jobs/303550431/",
"logs": [
{
"name": "builds-4h",
"name": "live_backing_log",
"url": "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/O5YBAWwxRfuZ_UlRJS5Rqg/runs/0/artifacts/public/logs/live_backing.log"
},
{

2
tests/ui/push-health/details/DetailsPanel_test.jsx
Просмотреть файл

@ -49,7 +49,7 @@ describe('DetailsPanel', () => {
'[taskcluster 2020-05-27 22:09:41.219Z] Task ID: WhN846qNQPin_jnGyF3w-g\n' +
'[taskcluster 2020-05-27 22:09:41.219Z] Worker ID: i-04f0e9fbffe5a9186\n',
);
fetchMock.get(getProjectUrl('/jobs/285852125/text_log_steps/', 'try'), [
fetchMock.get(getProjectUrl('/jobs/285852125/text_log_errors/', 'try'), [
{
id: 639432541,
errors: [

2
tests/ui/shared/FailureSummaryTab_test.jsx
Просмотреть файл

@ -37,7 +37,7 @@ describe('FailureSummaryTab', () => {
logViewerFullUrl="ber/baz"
addBug={() => {}}
pinJob={() => {}}
repoName="autoland"
repoName={repoName}
/>
);

107
tests/webapp/api/test_jobs_api.py
Просмотреть файл

@ -5,7 +5,7 @@ from dateutil import parser
from django.urls import reverse
from rest_framework.status import HTTP_400_BAD_REQUEST
from treeherder.model.models import Job, TextLogError, TextLogStep
from treeherder.model.models import Job, TextLogError
@pytest.mark.parametrize(
@ -209,111 +209,10 @@ def test_job_detail_not_found(client, test_repository):
assert resp.status_code == 404
def test_text_log_steps_and_errors(client, test_job):
TextLogStep.objects.create(
job=test_job,
name='step1',
started=datetime.datetime.utcfromtimestamp(0),
finished=datetime.datetime.utcfromtimestamp(100),
started_line_number=1,
finished_line_number=100,
result=TextLogStep.SUCCESS,
)
step2 = TextLogStep.objects.create(
job=test_job,
name='step2',
started=datetime.datetime.utcfromtimestamp(101),
finished=datetime.datetime.utcfromtimestamp(200),
started_line_number=101,
finished_line_number=200,
result=TextLogStep.TEST_FAILED,
)
TextLogError.objects.create(job=test_job, step=step2, line='failure 1', line_number=101)
TextLogError.objects.create(job=test_job, step=step2, line='failure 2', line_number=102)
resp = client.get(
reverse(
"jobs-text-log-steps", kwargs={"project": test_job.repository.name, "pk": test_job.id}
)
)
assert resp.status_code == 200
assert resp.json() == [
{
'errors': [],
'finished': '1970-01-01T00:01:40',
'finished_line_number': 100,
'id': 1,
'name': 'step1',
'result': 'success',
'started': '1970-01-01T00:00:00',
'started_line_number': 1,
},
{
'errors': [
{
'id': 1,
'job': 1,
'line': 'failure 1',
'line_number': 101,
'bug_suggestions': {
'search': 'failure 1',
'search_terms': ['failure 1'],
'bugs': {'open_recent': [], 'all_others': []},
'line_number': 101,
},
'metadata': None,
'matches': [],
'classified_failures': [],
},
{
'id': 2,
'job': 1,
'line': 'failure 2',
'line_number': 102,
'bug_suggestions': {
'search': 'failure 2',
'search_terms': ['failure 2'],
'bugs': {'open_recent': [], 'all_others': []},
'line_number': 102,
},
'metadata': None,
'matches': [],
'classified_failures': [],
},
],
'finished': '1970-01-01T00:03:20',
'finished_line_number': 200,
'id': 2,
'name': 'step2',
'result': 'testfailed',
'started': '1970-01-01T00:01:41',
'started_line_number': 101,
},
]
def test_text_log_errors(client, test_job):
TextLogStep.objects.create(
job=test_job,
name='step1',
started=datetime.datetime.utcfromtimestamp(0),
finished=datetime.datetime.utcfromtimestamp(100),
started_line_number=1,
finished_line_number=100,
result=TextLogStep.SUCCESS,
)
step2 = TextLogStep.objects.create(
job=test_job,
name='step2',
started=datetime.datetime.utcfromtimestamp(101),
finished=datetime.datetime.utcfromtimestamp(200),
started_line_number=101,
finished_line_number=200,
result=TextLogStep.TEST_FAILED,
)
TextLogError.objects.create(job=test_job, step=step2, line='failure 1', line_number=101)
TextLogError.objects.create(job=test_job, step=step2, line='failure 2', line_number=102)
TextLogError.objects.create(job=test_job, line='failure 1', line_number=101)
TextLogError.objects.create(job=test_job, line='failure 2', line_number=102)
resp = client.get(
reverse(
"jobs-text-log-errors", kwargs={"project": test_job.repository.name, "pk": test_job.id}

4
treeherder/autoclassify/autoclassify.py
Просмотреть файл

@ -50,8 +50,8 @@ def match_errors(job, matchers=None):
return
all_errors = set(
TextLogError.objects.filter(step__job=job, classified_failures=None).prefetch_related(
'step', '_metadata', '_metadata__failure_line'
TextLogError.objects.filter(job=job, classified_failures=None).prefetch_related(
'_metadata', '_metadata__failure_line'
)
)
errors = [t for t in all_errors if t.metadata and t.metadata.failure_line]

4
treeherder/autoclassify/matchers.py
Просмотреть файл

@ -32,7 +32,7 @@ def precise_matcher(text_log_error):
}
qwargs = Q(text_log_error___metadata__best_classification=None) & (
Q(text_log_error___metadata__best_is_verified=True)
| Q(text_log_error__step__job=text_log_error.step.job)
| Q(text_log_error__job=text_log_error.job)
)
qs = (
TextLogErrorMatch.objects.filter(**f)
@ -74,7 +74,7 @@ def crash_signature_matcher(text_log_error):
}
qwargs = Q(text_log_error___metadata__best_classification=None) & (
Q(text_log_error___metadata__best_is_verified=True)
| Q(text_log_error__step__job=text_log_error.step.job)
| Q(text_log_error__job=text_log_error.job)
)
qs = (
TextLogErrorMatch.objects.filter(**f)

2
treeherder/config/settings.py
Просмотреть файл

@ -423,7 +423,7 @@ BZ_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
COMMENTER_API_KEY = env("BUG_COMMENTER_API_KEY", default=None)
# Log Parsing
PARSER_MAX_STEP_ERROR_LINES = 100
MAX_ERROR_LINES = 100
FAILURE_LINES_CUTOFF = 35
# Perfherder

40
treeherder/etl/artifact.py
Просмотреть файл

@ -1,6 +1,5 @@
import logging
import dateutil.parser
import simplejson as json
from django.db import transaction
from django.db.utils import IntegrityError
@ -8,7 +7,7 @@ from django.db.utils import IntegrityError
from treeherder.etl.perf import store_performance_artifact
from treeherder.etl.text import astral_filter
from treeherder.model import error_summary
from treeherder.model.models import Job, TextLogError, TextLogStep
from treeherder.model.models import Job, TextLogError
logger = logging.getLogger(__name__)
@ -17,38 +16,15 @@ def store_text_log_summary_artifact(job, text_log_summary_artifact):
"""
Store the contents of the text log summary artifact
"""
step_data = json.loads(text_log_summary_artifact['blob'])['step_data']
result_map = {v: k for (k, v) in TextLogStep.RESULTS}
errors = json.loads(text_log_summary_artifact['blob'])['errors']
with transaction.atomic():
for step in step_data['steps']:
name = step['name'][: TextLogStep._meta.get_field('name').max_length]
# process start/end times if we have them
# we currently don't support timezones in treeherder, so
# just ignore that when importing/updating the bug to avoid
# a ValueError (though by default the text log summaries
# we produce should have time expressed in UTC anyway)
time_kwargs = {}
for tkey in ('started', 'finished'):
if step.get(tkey):
time_kwargs[tkey] = dateutil.parser.parse(step[tkey], ignoretz=True)
log_step = TextLogStep.objects.create(
job=job,
started_line_number=step['started_linenumber'],
finished_line_number=step['finished_linenumber'],
name=name,
result=result_map[step['result']],
**time_kwargs,
)
if step.get('errors'):
for error in step['errors']:
TextLogError.objects.create(
job=job,
step=log_step,
line_number=error['linenumber'],
line=astral_filter(error['line']),
for error in errors:
obj, created = TextLogError.objects.get_or_create(
job=job, line_number=error['linenumber'], line=astral_filter(error['line']),
)
if not created:
logger.warning('duplicate error lines processed for job %s', job.id)
# get error summary immediately (to warm the cache)
error_summary.get_error_summary(job)

12
treeherder/log_parser/artifactbuildercollection.py
Просмотреть файл

@ -4,7 +4,7 @@ import newrelic.agent
from treeherder.utils.http import make_request
from .artifactbuilders import BuildbotLogViewArtifactBuilder, BuildbotPerformanceDataArtifactBuilder
from .artifactbuilders import LogViewerArtifactBuilder, PerformanceDataArtifactBuilder
from .parsers import EmptyPerformanceData
logger = logging.getLogger(__name__)
@ -39,13 +39,13 @@ ArtifactBuilderBase
* parser
* Passes lines the ``Parser``
BuildbotLogViewArtifactBuilder
LogViewerArtifactBuilder
-------------
* Parses out content for use in a visual Log Parser
* Parsers:
* StepParser, which has its own ErrorParser
* ErrorParser
BuildbotPerformanceDataArtifactBuilder
PerformanceDataArtifactBuilder
-------------
* Builds an artifact from performance data
* Parsers:
@ -72,8 +72,8 @@ BuildbotPerformanceDataArtifactBuilder
else:
# use the defaults
self.builders = [
BuildbotLogViewArtifactBuilder(url=self.url),
BuildbotPerformanceDataArtifactBuilder(url=self.url),
LogViewerArtifactBuilder(url=self.url),
PerformanceDataArtifactBuilder(url=self.url),
]
def parse(self):

10
treeherder/log_parser/artifactbuilders.py
Просмотреть файл

@ -1,13 +1,13 @@
import logging
from .parsers import PerformanceParser, StepParser
from .parsers import PerformanceParser, ErrorParser
logger = logging.getLogger(__name__)
class ArtifactBuilderBase:
"""
Base class for all Buildbot log parsers.
Base class for all log parsers.
The child class will be designed to create a particular type of artifact.
@ -61,17 +61,17 @@ class ArtifactBuilderBase:
return self.artifact
class BuildbotLogViewArtifactBuilder(ArtifactBuilderBase):
class LogViewerArtifactBuilder(ArtifactBuilderBase):
"""Makes the artifact for the structured log viewer."""
def __init__(self, url=None):
"""Construct artifact builder for the log viewer"""
super().__init__(url)
self.parser = StepParser()
self.parser = ErrorParser()
self.name = "text_log_summary"
class BuildbotPerformanceDataArtifactBuilder(ArtifactBuilderBase):
class PerformanceDataArtifactBuilder(ArtifactBuilderBase):
"""Makes the artifact for performance data."""
def __init__(self, url=None):

2
treeherder/log_parser/crossreference.py
Просмотреть файл

@ -39,7 +39,7 @@ def crossreference_job(job):
@transaction.atomic
def _crossreference(job):
failure_lines = FailureLine.objects.filter(job_guid=job.guid)
text_log_errors = TextLogError.objects.filter(step__job=job).order_by('line_number')
text_log_errors = TextLogError.objects.filter(job=job).order_by('line_number')
if not failure_lines and text_log_errors:
return False

224
treeherder/log_parser/parsers.py
Просмотреть файл

@ -39,227 +39,6 @@ class ParserBase:
return self.artifact
class StepParser(ParserBase):
"""
Parse out individual job steps within a log.
Step format:
"steps": [
{
"errors": [],
"name": "set props: master", # the name of the process on start line
"started": "2013-06-05 12:39:57.838527",
"started_linenumber": 8,
"finished_linenumber": 10,
"finished": "2013-06-05 12:39:57.839226",
"result": 0
},
...
]
"""
# Matches the half-dozen 'key: value' header lines printed at the start of each
# Buildbot job log. The list of keys are taken from:
# https://hg.mozilla.org/build/buildbotcustom/file/644c3860300a/bin/log_uploader.py#l126
RE_HEADER_LINE = re.compile(
r'(?:builder|slave|starttime|results|buildid|builduid|revision): .*'
)
# Step marker lines, eg:
# ========= Started foo (results: 0, elapsed: 0 secs) (at 2015-08-17 02:33:56.353866) =========
# ========= Finished foo (results: 0, elapsed: 0 secs) (at 2015-08-17 02:33:56.354301) =========
RE_STEP_MARKER = re.compile(
r'={9} (?P<marker_type>Started|Finished) (?P<name>.*?) '
r'\(results: (?P<result_code>\d+), elapsed: .*?\) '
r'\(at (?P<timestamp>.*?)\)'
)
# Legacy result code to name mapping inherited from buildbot (duplicated in TextLogStep)
# TODO: Likely remove this and step handling entirely now that Taskcluster doesn't have steps.
RESULT_DICT = {
0: "success",
1: "testfailed",
2: "busted",
3: "skipped",
4: "exception",
5: "retry",
6: "usercancel",
7: "superseded",
}
STATES = {
# The initial state until we record the first step.
"awaiting_first_step": 0,
# We've started a step, but not yet seen the end of it.
"step_in_progress": 1,
# We've seen the end of the previous step.
"step_finished": 2,
}
# date format in a step started/finished header
DATE_FORMAT = '%Y-%m-%d %H:%M:%S.%f'
def __init__(self):
"""Setup the artifact to hold the header lines."""
super().__init__("step_data")
self.stepnum = -1
self.artifact = {"steps": [], "errors_truncated": False}
self.sub_parser = ErrorParser()
self.state = self.STATES['awaiting_first_step']
def parse_line(self, line, lineno):
"""Parse a single line of the log.
We have to handle both buildbot style logs as well as Taskcluster logs. The latter
attempt to emulate the buildbot logs, but don't accurately do so, partly due
to the way logs are generated in Taskcluster (ie: on the workers themselves).
Buildbot logs:
builder: ...
slave: ...
starttime: ...
results: ...
buildid: ...
builduid: ...
revision: ...
======= <step START marker> =======
<step log output>
======= <step FINISH marker> =======
======= <step START marker> =======
<step log output>
======= <step FINISH marker> =======
Taskcluster logs (a worst-case example):
<log output outside a step>
======= <step START marker> =======
<step log output>
======= <step FINISH marker> =======
<log output outside a step>
======= <step START marker> =======
<step log output with no following finish marker>
As can be seen above, Taskcluster logs can have (a) log output that falls between
step markers, and (b) content at the end of the log, that is not followed by a
final finish step marker. We handle this by creating generic placeholder steps to
hold the log output that is not enclosed by step markers, and then by cleaning up
the final step in finish_parse() once all lines have been parsed.
"""
if not line.strip():
# Skip whitespace-only lines, since they will never contain an error line,
# so are not of interest. This also avoids creating spurious unnamed steps
# (which occurs when we find content outside of step markers) for the
# newlines that separate the steps in Buildbot logs.
return
if self.state == self.STATES['awaiting_first_step'] and self.RE_HEADER_LINE.match(line):
# The "key: value" job metadata header lines that appear at the top of
# Buildbot logs would result in the creation of an unnamed step at the
# start of the job, unless we skip them. (Which is not desired, since
# the lines are metadata and not test/build output.)
return
step_marker_match = self.RE_STEP_MARKER.match(line)
if not step_marker_match:
# This is a normal log line, rather than a step marker. (The common case.)
if self.state != self.STATES['step_in_progress']:
# We don't have an in-progress step, so need to start one, even though this
# isn't a "step started" marker line. We therefore create a new generic step,
# since we have no way of finding out the step metadata. This case occurs
# for the Taskcluster logs where content can fall between step markers.
self.start_step(lineno)
# Parse the line for errors, which if found, will be associated with the current step.
self.sub_parser.parse_line(line, lineno)
return
# This is either a "step started" or "step finished" marker line, eg:
# ========= Started foo (results: 0, elapsed: 0 secs) (at 2015-08-17 02:33:56.353866) =========
# ========= Finished foo (results: 0, elapsed: 0 secs) (at 2015-08-17 02:33:56.354301) =========
if step_marker_match.group('marker_type') == 'Started':
if self.state == self.STATES['step_in_progress']:
# We're partway through a step (ie: haven't seen a "step finished" marker line),
# but have now reached the "step started" marker for the next step. Before we
# can start the new step, we have to clean up the previous one - albeit using
# generic step metadata, since there was no "step finished" marker. This occurs
# in Taskcluster's logs when content falls between the step marker lines.
self.end_step(lineno)
# Start a new step using the extracted step metadata.
self.start_step(
lineno,
name=step_marker_match.group('name'),
timestamp=step_marker_match.group('timestamp'),
)
return
# This is a "step finished" marker line.
if self.state != self.STATES['step_in_progress']:
# We're not in the middle of a step, so can't finish one. Just ignore the marker line.
return
# Close out the current step using the extracted step metadata.
self.end_step(
lineno,
timestamp=step_marker_match.group('timestamp'),
result_code=int(step_marker_match.group('result_code')),
)
def start_step(self, lineno, name="Unnamed step", timestamp=None):
"""Create a new step and update the state to reflect we're now in the middle of a step."""
self.state = self.STATES['step_in_progress']
self.stepnum += 1
self.steps.append(
{"name": name, "started": timestamp, "started_linenumber": lineno, "errors": [],}
)
def end_step(self, lineno, timestamp=None, result_code=None):
"""Fill in the current step's summary and update the state to show the current step has ended."""
self.state = self.STATES['step_finished']
step_errors = self.sub_parser.get_artifact()
step_error_count = len(step_errors)
if step_error_count > settings.PARSER_MAX_STEP_ERROR_LINES:
step_errors = step_errors[: settings.PARSER_MAX_STEP_ERROR_LINES]
self.artifact["errors_truncated"] = True
self.current_step.update(
{
"finished": timestamp,
"finished_linenumber": lineno,
# Whilst the result code is present on both the start and end buildbot-style step
# markers, for Taskcluster logs the start marker line lies about the result, since
# the log output is unbuffered, so Taskcluster does not know the real result at
# that point. As such, we only set the result when ending a step.
"result": self.RESULT_DICT.get(result_code, "unknown"),
"errors": step_errors,
}
)
# reset the sub_parser for the next step
self.sub_parser.clear()
def finish_parse(self, last_lineno_seen):
"""Clean-up/summary tasks run at the end of parsing."""
if self.state == self.STATES['step_in_progress']:
# We've reached the end of the log without seeing the final "step finish"
# marker, which would normally have triggered updating the step. As such we
# must manually close out the current step, so things like result, finish
# time are set for it. This ensures that the error summary for Taskcluster
# infra failures actually lists the error that occurs at the
# end of the log.
self.end_step(last_lineno_seen)
@property
def steps(self):
"""Return the list of steps in the artifact"""
return self.artifact["steps"]
@property
def current_step(self):
"""Return the current step in the artifact"""
return self.steps[self.stepnum]
class ErrorParser(ParserBase):
"""A generic error detection sub-parser"""
@ -339,6 +118,9 @@ class ErrorParser(ParserBase):
def parse_line(self, line, lineno):
"""Check a single line for an error. Keeps track of the linenumber"""
if len(self.artifact) >= settings.MAX_ERROR_LINES:
return
# TaskCluster logs are a bit wonky.
#
# TaskCluster logs begin with output coming from TaskCluster itself,

10
treeherder/log_parser/tasks.py
Просмотреть файл

@ -32,7 +32,7 @@ def parse_logs(job_id, job_log_ids, priority):
parser_tasks = {
"errorsummary_json": store_failure_lines,
"live_backing_log": parse_unstructured_log,
"live_backing_log": post_log_artifacts,
}
# We don't want to stop parsing logs for most Exceptions however we still
@ -95,14 +95,6 @@ def parse_logs(job_id, job_log_ids, priority):
job.save()
def parse_unstructured_log(job_log):
"""
Call ArtifactBuilderCollection on the given job.
"""
logger.debug('Running parse_unstructured_log for job %s', job_log.job.id)
post_log_artifacts(job_log)
def store_failure_lines(job_log):
"""Store the failure lines from a log corresponding to the structured
errorsummary file."""

8
treeherder/model/error_summary.py
Просмотреть файл

@ -1,5 +1,6 @@
import logging
import re
import newrelic.agent
from django.core.cache import cache
@ -32,7 +33,7 @@ def get_error_summary(job):
# don't cache or do anything if we have no text log errors to get
# results for
errors = TextLogError.objects.filter(step__job=job)
errors = TextLogError.objects.filter(job=job)
if not errors:
return []
@ -40,7 +41,12 @@ def get_error_summary(job):
term_cache = {}
error_summary = [bug_suggestions_line(err, term_cache) for err in errors]
try:
cache.set(cache_key, error_summary, BUG_SUGGESTION_CACHE_TIMEOUT)
except Exception as e:
newrelic.agent.record_custom_event('error caching error_summary for job', job.id)
logger.error('error caching error_summary for job %s: %s', job.id, e, exc_info=True)
return error_summary

13
treeherder/model/models.py
Просмотреть файл

@ -566,7 +566,7 @@ class Job(models.Model):
return False
classified_error_count = TextLogError.objects.filter(
_metadata__best_classification__isnull=False, step__job=self
_metadata__best_classification__isnull=False, job=self
).count()
if classified_error_count == 0:
@ -586,7 +586,7 @@ class Job(models.Model):
instances are set to True.
"""
unverified_errors = TextLogError.objects.filter(
_metadata__best_is_verified=False, step__job=self
_metadata__best_is_verified=False, job=self
).count()
if unverified_errors:
@ -621,7 +621,7 @@ class Job(models.Model):
"""
If this Job has a single TextLogError line, return that TextLogError.
Some Jobs only have one related [via TextLogStep] TextLogError. This
Some Jobs only have one related TextLogError. This
method checks if this Job is one of those (returning None if not) by:
* checking the number of related TextLogErrors
* counting the number of search results for the single TextLogError
@ -631,7 +631,7 @@ class Job(models.Model):
If all these checks pass the TextLogError is returned, any failure returns None.
"""
try:
text_log_error = TextLogError.objects.get(step__job=self)
text_log_error = TextLogError.objects.get(job=self)
except (TextLogError.DoesNotExist, TextLogError.MultipleObjectsReturned):
return None
@ -912,8 +912,7 @@ class JobNote(models.Model):
# TODO: Decide whether this should change now that we're no longer mirroring.
bug_numbers = set(
ClassifiedFailure.objects.filter(
best_for_errors__text_log_error__step__job=job,
best_for_errors__best_is_verified=True,
best_for_errors__text_log_error__job=job, best_for_errors__best_is_verified=True,
)
.exclude(bug_number=None)
.exclude(bug_number=0)
@ -1267,7 +1266,7 @@ class TextLogError(models.Model):
unique_together = ('step', 'line_number')
def __str__(self):
return "{0} {1}".format(self.id, self.step.job.id)
return "{0} {1}".format(self.id, self.job.id)
@property
def metadata(self):

4
treeherder/webapp/api/intermittents_view.py
Просмотреть файл

@ -81,9 +81,9 @@ class FailuresByBug(generics.ListAPIView):
)
lines = TextLogError.objects.filter(
step__job_id__in=self.queryset.values_list('job_id', flat=True),
job_id__in=self.queryset.values_list('job_id', flat=True),
line__contains='TEST-UNEXPECTED-FAIL',
).values_list('step__job_id', 'line')
).values_list('job_id', 'line')
grouped_lines = defaultdict(list)
for job_id, line in lines:

5
treeherder/webapp/api/jobs.py
Просмотреть файл

@ -375,6 +375,7 @@ class JobsProjectViewSet(viewsets.ViewSet):
return Response(response_body)
# TODO remove
@action(detail=True, methods=['get'])
def text_log_steps(self, request, project, pk=None):
"""
@ -397,14 +398,14 @@ class JobsProjectViewSet(viewsets.ViewSet):
@action(detail=True, methods=['get'])
def text_log_errors(self, request, project, pk=None):
"""
Gets a list of steps associated with this job
Gets a list of error lines associated with this job
"""
try:
job = Job.objects.get(repository__name=project, id=pk)
except Job.DoesNotExist:
return Response("No job with id: {0}".format(pk), status=HTTP_404_NOT_FOUND)
textlog_errors = (
TextLogError.objects.filter(step__job=job)
TextLogError.objects.filter(job=job)
.select_related("_metadata", "_metadata__failure_line")
.prefetch_related("classified_failures", "matches")
.order_by('id')

2
ui/helpers/url.js
Просмотреть файл

@ -31,6 +31,8 @@ export const repoEndpoint = '/repository/';
export const tcAuthCallbackUrl = '/taskcluster-auth.html';
export const textLogErrorsEndpoint = '/text_log_errors/';
export const getRunnableJobsURL = function getRunnableJobsURL(
decisionTask,
rootUrl,

17
ui/job-view/details/tabs/SimilarJobsTab.jsx
Просмотреть файл

@ -8,11 +8,12 @@ import { Button } from 'reactstrap';
import { thMaxPushFetchSize } from '../../../helpers/constants';
import { toDateStr, toShortDateStr } from '../../../helpers/display';
import { addAggregateFields, getBtnClass } from '../../../helpers/job';
import { getJobsUrl } from '../../../helpers/url';
import { getJobsUrl, textLogErrorsEndpoint } from '../../../helpers/url';
import JobModel from '../../../models/job';
import PushModel from '../../../models/push';
import TextLogStepModel from '../../../models/textLogStep';
import { notify } from '../../redux/stores/notifications';
import { getProjectJobUrl } from '../../../helpers/location';
import { getData } from '../../../helpers/http';
class SimilarJobsTab extends React.Component {
constructor(props) {
@ -124,20 +125,20 @@ class SimilarJobsTab extends React.Component {
showJobInfo = (job) => {
const { repoName, classificationMap } = this.props;
JobModel.get(repoName, job.id).then((nextJob) => {
JobModel.get(repoName, job.id).then(async (nextJob) => {
addAggregateFields(nextJob);
nextJob.failure_classification =
classificationMap[nextJob.failure_classification_id];
// retrieve the list of error lines
TextLogStepModel.get(nextJob.id).then((textLogSteps) => {
nextJob.error_lines = textLogSteps.reduce(
(acc, step) => [...acc, ...step.errors],
[],
const { data, failureStatus } = await getData(
getProjectJobUrl(textLogErrorsEndpoint, nextJob.id),
);
if (!failureStatus && data.length) {
nextJob.error_lines = data;
}
this.setState({ selectedSimilarJob: nextJob });
});
});
};
toggleFilter = (filterField) => {

28
ui/logviewer/App.jsx
Просмотреть файл

@ -4,14 +4,23 @@ import { LazyLog } from 'react-lazylog';
import isEqual from 'lodash/isEqual';
import { Collapse } from 'reactstrap';
import { getAllUrlParams, getUrlParam, setUrlParam } from '../helpers/location';
import {
getAllUrlParams,
getUrlParam,
setUrlParam,
getProjectJobUrl,
} from '../helpers/location';
import { scrollToLine } from '../helpers/utils';
import { isReftest } from '../helpers/job';
import { getJobsUrl, getReftestUrl, getArtifactsUrl } from '../helpers/url';
import {
getJobsUrl,
getReftestUrl,
getArtifactsUrl,
textLogErrorsEndpoint,
} from '../helpers/url';
import { getData } from '../helpers/http';
import JobModel from '../models/job';
import PushModel from '../models/push';
import TextLogStepModel from '../models/textLogStep';
import JobDetails from '../shared/JobDetails';
import JobInfo from '../shared/JobInfo';
import RepositoryModel from '../models/repository';
@ -50,7 +59,7 @@ class App extends React.PureComponent {
};
}
componentDidMount() {
async componentDidMount() {
const { repoName, jobId } = this.state;
const repoPromise = RepositoryModel.getList();
@ -148,9 +157,12 @@ class App extends React.PureComponent {
});
});
TextLogStepModel.get(jobId).then((textLogSteps) => {
const stepErrors = textLogSteps.length ? textLogSteps[0].errors : [];
const errors = stepErrors.map((error) => ({
const { data, failureStatus } = await getData(
getProjectJobUrl(textLogErrorsEndpoint, jobId),
);
if (!failureStatus && data.length) {
const errors = data.map((error) => ({
line: error.line,
lineNumber: error.line_number + 1,
}));
@ -163,7 +175,7 @@ class App extends React.PureComponent {
errorLinesCss(errors);
this.setState({ errors });
this.setSelectedLine(highlight, true);
});
}
}
onHighlight = (range) => {

20
ui/models/textLogErrors.js
Просмотреть файл

@ -1,20 +0,0 @@
import { getApiUrl } from '../helpers/url';
import { update } from '../helpers/http';
const uri = getApiUrl('/text-log-error/');
export default class TextLogErrorsModel {
constructor(data) {
if (data.metadata === null) {
data.metadata = {};
}
Object.assign(this, data);
}
static verifyMany(body) {
if (!body.length) {
return Promise.resolve();
}
return update(uri, body);
}
}

9
ui/models/textLogStep.js
Просмотреть файл

@ -1,9 +0,0 @@
import { getProjectJobUrl } from '../helpers/location';
export default class TextLogStepModel {
static get(jobId) {
return fetch(getProjectJobUrl('/text_log_steps/', jobId)).then((resp) =>
resp.json(),
);
}
}

24
ui/shared/tabs/LogviewerTab.jsx
Просмотреть файл

@ -4,9 +4,10 @@ import { LazyLog } from 'react-lazylog';
import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
import { faExpand, faFileAlt } from '@fortawesome/free-solid-svg-icons';
import TextLogStepModel from '../../models/textLogStep';
import { getLogViewerUrl } from '../../helpers/url';
import { getLogViewerUrl, textLogErrorsEndpoint } from '../../helpers/url';
import { errorLinesCss } from '../../helpers/display';
import { getData } from '../../helpers/http';
import { getProjectJobUrl } from '../../helpers/location';
class LogviewerTab extends React.PureComponent {
constructor(props) {
@ -17,30 +18,35 @@ class LogviewerTab extends React.PureComponent {
};
}
componentDidMount() {
async componentDidMount() {
const {
selectedTaskFull: { id: taskId },
selectedTaskFull: { id: jobId },
} = this.props;
TextLogStepModel.get(taskId).then((textLogSteps) => {
const stepErrors = textLogSteps.length ? textLogSteps[0].errors : [];
const logErrors = stepErrors.map((error) => ({
const { data, failureStatus } = await getData(
getProjectJobUrl(textLogErrorsEndpoint, jobId),
);
if (!failureStatus && data.length) {
const logErrors = data.map((error) => ({
line: error.line,
lineNumber: error.line_number + 1,
}));
const firstErrorLineNumber = logErrors.length
? [logErrors[0].lineNumber]
: null;
errorLinesCss(logErrors);
this.setState({ highlight: firstErrorLineNumber });
});
}
}
render() {
const { selectedTaskFull, repoName } = this.props;
const { highlight } = this.state;
const { url } = selectedTaskFull.logs[0];
const { url } = selectedTaskFull.logs.find(
(log) => log.name === 'live_backing_log',
);
return (
<div className="h-100 w-100" aria-label="Log">

10
ui/shared/tabs/failureSummary/ErrorsList.jsx
Просмотреть файл

@ -2,11 +2,9 @@ import React from 'react';
import PropTypes from 'prop-types';
export default function ErrorsList(props) {
const errorListItem = props.errors.map((error, key) => (
<li
key={key} // eslint-disable-line react/no-array-index-key
>
{error.name} : {error.result}.
const errorListItem = props.errors.map((error) => (
<li key={error.line_number}>
{error.line}
<a
title="Open in Log Viewer"
target="_blank"
@ -22,7 +20,7 @@ export default function ErrorsList(props) {
<li>
No Bug Suggestions Available.
<br />
<span className="font-weight-bold">Unsuccessful Execution Steps</span>
<span className="font-weight-bold">Failure Lines</span>
<ul>{errorListItem}</ul>
</li>
);

23
ui/shared/tabs/failureSummary/FailureSummaryTab.jsx
Просмотреть файл

@ -9,10 +9,12 @@ import {
getBugUrl,
getLogViewerUrl,
getReftestUrl,
textLogErrorsEndpoint,
} from '../../../helpers/url';
import BugFiler from '../../BugFiler';
import BugSuggestionsModel from '../../../models/bugSuggestions';
import TextLogStepModel from '../../../models/textLogStep';
import { getData } from '../../../helpers/http';
import { getProjectJobUrl } from '../../../helpers/location';
import ErrorsList from './ErrorsList';
import ListItem from './ListItem';
@ -77,7 +79,7 @@ class FailureSummaryTab extends React.Component {
if (!selectedJob) {
return;
}
BugSuggestionsModel.get(selectedJob.id).then((suggestions) => {
BugSuggestionsModel.get(selectedJob.id).then(async (suggestions) => {
suggestions.forEach((suggestion) => {
suggestion.bugs.too_many_open_recent =
suggestion.bugs.open_recent.length > thBugSuggestionLimit;
@ -98,20 +100,21 @@ class FailureSummaryTab extends React.Component {
// the log (we can do this asynchronously, it should normally be
// fast)
if (!suggestions.length) {
TextLogStepModel.get(selectedJob.id).then((textLogSteps) => {
const errors = textLogSteps
.filter((step) => step.result !== 'success')
.map((step) => ({
name: step.name,
result: step.result,
const { data, failureStatus } = await getData(
getProjectJobUrl(textLogErrorsEndpoint, selectedJob.id),
);
if (!failureStatus && data.length) {
const errors = data.map((error) => ({
line: error.line,
line_number: error.line_number,
logViewerUrl: getLogViewerUrl(
selectedJob.id,
repoName,
step.finished_line_number,
error.line_number,
),
}));
this.setState({ errors });
});
}
}
this.setState({ bugSuggestionsLoading: false, suggestions }, () => {