зеркало из https://github.com/mozilla/treeherder.git
fixes to handle single job is blobs
This commit is contained in:
Cameron Dawson
Родитель
356c2fab9a
Коммит
be0fab3a38
|
|
@ -23,40 +23,16 @@ def test_disconnect(jm):
|
|||
assert not jm.get_jobs_dhub().connection["master_host"]["con_obj"].open
|
||||
|
||||
|
||||
def test_ingest_single_sample_job(jm, sample_data):
|
||||
"""Process a single job structure in the job_data.txt file"""
|
||||
blob = sample_data.job_data[0]
|
||||
jm.store_job_data(json.dumps(blob))
|
||||
jobs = jm.process_objects(1)
|
||||
# import time
|
||||
# time.sleep(30)
|
||||
assert len(jobs) == 1
|
||||
job_id = jobs[0]
|
||||
|
||||
exp_job = clean_job_blob_dict(blob["jobs"][0])
|
||||
act_job = JobDictBuilder(jm, job_id).as_dict()
|
||||
|
||||
assert exp_job == act_job, diff_dict(exp_job, act_job)
|
||||
|
||||
complete_count = jm.get_os_dhub().execute(
|
||||
proc="objectstore_test.counts.complete")[0]["complete_count"]
|
||||
loading_count = jm.get_os_dhub().execute(
|
||||
proc="objectstore_test.counts.loading")[0]["loading_count"]
|
||||
|
||||
assert complete_count == 1
|
||||
assert loading_count == 0
|
||||
|
||||
|
||||
@slow
|
||||
def test_ingest_all_sample_jobs(jm, sample_data):
|
||||
def do_job_ingestion(jm, job_data):
|
||||
"""
|
||||
Process each job structure in the job_data.txt file and verify.
|
||||
Test ingesting job blobs. ``job_data`` maybe different sizes.
|
||||
|
||||
Process each job structure in the job_data and verify.
|
||||
|
||||
This rebuilds the JSON blob (for the most part) and compares that
|
||||
everything was stored correctly.
|
||||
|
||||
"""
|
||||
job_data = sample_data.job_data
|
||||
for blob in job_data:
|
||||
jm.store_job_data(json.dumps(blob))
|
||||
jobs = jm.process_objects(1)
|
||||
|
|
@ -64,7 +40,7 @@ def test_ingest_all_sample_jobs(jm, sample_data):
|
|||
job_id = jobs[0]
|
||||
|
||||
# verify the job data
|
||||
exp_job = clean_job_blob_dict(blob["jobs"][0])
|
||||
exp_job = clean_job_blob_dict(blob["job"])
|
||||
act_job = JobDictBuilder(jm, job_id).as_dict()
|
||||
assert exp_job == act_job, diff_dict(exp_job, act_job)
|
||||
|
||||
|
|
@ -82,6 +58,22 @@ def test_ingest_all_sample_jobs(jm, sample_data):
|
|||
assert loading_count == 0
|
||||
|
||||
|
||||
def test_ingest_single_sample_job(jm, sample_data):
|
||||
"""Process a single job structure in the job_data.txt file"""
|
||||
job_data = sample_data.job_data[:0]
|
||||
do_job_ingestion(jm, job_data)
|
||||
|
||||
|
||||
@slow
|
||||
def test_ingest_all_sample_jobs(jm, sample_data):
|
||||
"""
|
||||
Process each job structure in the job_data.txt file and verify.
|
||||
|
||||
"""
|
||||
job_data = sample_data.job_data
|
||||
do_job_ingestion(jm, job_data)
|
||||
|
||||
|
||||
class SourceDictBuilder(object):
|
||||
"""Given a ``job_id``, rebuild the dictionary the source came from."""
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
{"sources": [{"commit_timestamp": 1366331469, "push_timestamp": 1366331469, "comments": "try: -b do -p macosx,macosx64,win32,linux,linux32 -u all -t none", "repository": "try", "revision": "12e26c5d0d57"}], "revision_hash": "28d50025d238a4410a844f11d6cec95cf2abf214", "jobs": [{"build_platform": {"platform": "OS X 10.6", "os_name": "mac", "architecture": "x86_64", "vm": false}, "submit_timestamp": 1366331470, "start_timestamp": 1366356690.0, "name": "reftest", "option_collection": {"debug": true}, "log_references": [{"url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/try-builds/mduan@mozilla.com-5cfd4cc28845/try-macosx64-debug/try_lion-debug_test-reftest-bm75-tests1-macosx-build37.txt.gz", "name": "unittest"}], "who": "mduan@mozilla.com", "reason": "scheduler", "artifact": {}, "machine_platform": {"platform": "OS X 10.6", "os_name": "mac", "architecture": "x86_64", "vm": false}, "machine": "talos-r4-lion-010", "state": "TODO", "result": 0, "job_guid": "8cc1cfe5e467d45f220bc7e63781a375b5582d48", "product_name": "firefox", "end_timestamp": "1366334018"}]}
|
||||
{"sources": [{"commit_timestamp": 1366331469, "push_timestamp": 1366331469, "comments": "try: -b do -p macosx,macosx64,win32,linux,linux32 -u all -t none", "repository": "try", "revision": "12e26c5d0d57"}], "revision_hash": "28d50025d238a4410a844f11d6cec95cf2abf214", "job": {"build_platform": {"platform": "OS X 10.6", "os_name": "mac", "architecture": "x86_64", "vm": false}, "submit_timestamp": 1366331470, "start_timestamp": 1366356690.0, "name": "reftest", "option_collection": {"debug": true}, "log_references": [{"url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/try-builds/mduan@mozilla.com-5cfd4cc28845/try-macosx64-debug/try_lion-debug_test-reftest-bm75-tests1-macosx-build37.txt.gz", "name": "unittest"}], "who": "mduan@mozilla.com", "reason": "scheduler", "artifact": {}, "machine_platform": {"platform": "OS X 10.6", "os_name": "mac", "architecture": "x86_64", "vm": false}, "machine": "talos-r4-lion-010", "state": "TODO", "result": 0, "job_guid": "8cc1cfe5e467d45f220bc7e63781a375b5582d48", "product_name": "firefox", "end_timestamp": "1366334018"}}
|
||||
{"sources": [{"commit_timestamp": 1366332511, "push_timestamp": 1366332511, "comments": "Bug 856351 - Paris binding for DragEvent, r=peterv", "repository": "mozilla-inbound", "revision": "4b5381804b7c"}], "revision_hash": "97932524871cb0306f587baab1f4200dae27dffc", "jobs": [{"build_platform": {"platform": "2.2", "os_name": "android", "architecture": "ARMv7", "vm": false}, "submit_timestamp": 1366332549, "start_timestamp": 1366357753.0, "name": "jsreftest-1", "option_collection": {"opt": true}, "log_references": [{"url": "http://ftp.mozilla.org/pub/mozilla.org/mobile/tinderbox-builds/mozilla-inbound-android-noion/1366326906/mozilla-inbound_tegra_android-noion_test-jsreftest-1-bm22-tests1-tegra-build2402.txt.gz", "name": "unittest"}], "who": "sendchange-unittest", "reason": "scheduler", "artifact": {}, "machine_platform": {"platform": "2.2", "os_name": "android", "architecture": "ARMv7", "vm": false}, "machine": "tegra-206", "state": "TODO", "result": 0, "job_guid": "6ff0cc5fb09aed3ef2c1beaa408a478a83ba2f78", "product_name": "mobile", "end_timestamp": "1366334021"}]}
|
||||
{"sources": [{"commit_timestamp": 1366332167, "push_timestamp": 1366332167, "comments": "Bug 856351 - Paris binding for DragEvent, r=peterv", "repository": "mozilla-inbound", "revision": "4b5381804b7c"}], "revision_hash": "97932524871cb0306f587baab1f4200dae27dffc", "jobs": [{"build_platform": {"platform": "2.2", "os_name": "android", "architecture": "ARMv7", "vm": false}, "submit_timestamp": 1366332191, "start_timestamp": 1366357394.0, "name": "plain-reftest-4", "option_collection": {"opt": true}, "log_references": [{"url": "http://ftp.mozilla.org/pub/mozilla.org/mobile/tinderbox-builds/mozilla-inbound-android-armv6/1366326906/mozilla-inbound_tegra_android-armv6_test-plain-reftest-4-bm19-tests1-tegra-build978.txt.gz", "name": "unittest"}], "who": "sendchange-unittest", "reason": "scheduler", "artifact": {}, "machine_platform": {"platform": "2.2", "os_name": "android", "architecture": "ARMv7", "vm": false}, "machine": "tegra-228", "state": "TODO", "result": 0, "job_guid": "a3c4ee0b561dd5a1c444b41424612d9bd6d4cd8d", "product_name": "mobile", "end_timestamp": "1366334024"}]}
|
||||
{"sources": [{"commit_timestamp": 1366332895, "push_timestamp": 1366332895, "comments": "Bug 861317 - nsChildView::WillPaint terribly inefficient when drawing to titlebar. r=bgirard", "repository": "mozilla-inbound", "revision": "e75ff759c5c1"}], "revision_hash": "57c1fbabe544656b4edb96ad2c5678fda8c5e46d", "jobs": [{"build_platform": {"platform": "WINNT 6.3", "os_name": "win", "architecture": "x86_64", "vm": false}, "submit_timestamp": 1366332906, "start_timestamp": 1366358108.0, "name": "chromez", "option_collection": {"opt": true}, "log_references": [{"url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/tinderbox-builds/mozilla-inbound-win32/1366328526/mozilla-inbound_win8_test-chromez-bm15-tests1-windows-build188.txt.gz", "name": "talos"}], "who": "sendchange", "reason": "scheduler", "artifact": {}, "machine_platform": {"platform": "WINNT 6.3", "os_name": "win", "architecture": "x86_64", "vm": false}, "machine": "t-w864-ix-079", "state": "TODO", "result": 0, "job_guid": "cd6e1a5dc81cc0279acf4a097ab9745ff45d708a", "product_name": "firefox", "end_timestamp": "1366334026"}]}
|
||||
|
|
|
|||
|
|
@ -185,7 +185,7 @@ class JobsModel(TreeherderModelBase):
|
|||
# set Job data
|
||||
|
||||
rdm = self.refdata_model
|
||||
job = data["jobs"]
|
||||
job = data["job"]
|
||||
|
||||
build_platform_id = rdm.get_or_create_build_platform(
|
||||
job["build_platform"]["os_name"],
|
||||
|
|
@ -258,7 +258,7 @@ class JobsModel(TreeherderModelBase):
|
|||
log_ref["url"]
|
||||
)
|
||||
|
||||
except KeyError:
|
||||
except (KeyError, JobDataError):
|
||||
# it is ok to have an empty or missing artifact
|
||||
pass
|
||||
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче