diff --git a/tests/model/derived/test_jobs_model.py b/tests/model/derived/test_jobs_model.py index 66d0891aa..2bca70db9 100644 --- a/tests/model/derived/test_jobs_model.py +++ b/tests/model/derived/test_jobs_model.py @@ -23,40 +23,16 @@ def test_disconnect(jm): assert not jm.get_jobs_dhub().connection["master_host"]["con_obj"].open -def test_ingest_single_sample_job(jm, sample_data): - """Process a single job structure in the job_data.txt file""" - blob = sample_data.job_data[0] - jm.store_job_data(json.dumps(blob)) - jobs = jm.process_objects(1) - # import time - # time.sleep(30) - assert len(jobs) == 1 - job_id = jobs[0] - - exp_job = clean_job_blob_dict(blob["jobs"][0]) - act_job = JobDictBuilder(jm, job_id).as_dict() - - assert exp_job == act_job, diff_dict(exp_job, act_job) - - complete_count = jm.get_os_dhub().execute( - proc="objectstore_test.counts.complete")[0]["complete_count"] - loading_count = jm.get_os_dhub().execute( - proc="objectstore_test.counts.loading")[0]["loading_count"] - - assert complete_count == 1 - assert loading_count == 0 - - -@slow -def test_ingest_all_sample_jobs(jm, sample_data): +def do_job_ingestion(jm, job_data): """ - Process each job structure in the job_data.txt file and verify. + Test ingesting job blobs. ``job_data`` maybe different sizes. + + Process each job structure in the job_data and verify. This rebuilds the JSON blob (for the most part) and compares that everything was stored correctly. """ - job_data = sample_data.job_data for blob in job_data: jm.store_job_data(json.dumps(blob)) jobs = jm.process_objects(1) @@ -64,7 +40,7 @@ def test_ingest_all_sample_jobs(jm, sample_data): job_id = jobs[0] # verify the job data - exp_job = clean_job_blob_dict(blob["jobs"][0]) + exp_job = clean_job_blob_dict(blob["job"]) act_job = JobDictBuilder(jm, job_id).as_dict() assert exp_job == act_job, diff_dict(exp_job, act_job) @@ -82,6 +58,22 @@ def test_ingest_all_sample_jobs(jm, sample_data): assert loading_count == 0 +def test_ingest_single_sample_job(jm, sample_data): + """Process a single job structure in the job_data.txt file""" + job_data = sample_data.job_data[:0] + do_job_ingestion(jm, job_data) + + +@slow +def test_ingest_all_sample_jobs(jm, sample_data): + """ + Process each job structure in the job_data.txt file and verify. + + """ + job_data = sample_data.job_data + do_job_ingestion(jm, job_data) + + class SourceDictBuilder(object): """Given a ``job_id``, rebuild the dictionary the source came from.""" diff --git a/tests/sample_data/job_data.txt b/tests/sample_data/job_data.txt index 2fe6d588c..43d3a148a 100644 --- a/tests/sample_data/job_data.txt +++ b/tests/sample_data/job_data.txt @@ -1,4 +1,4 @@ -{"sources": [{"commit_timestamp": 1366331469, "push_timestamp": 1366331469, "comments": "try: -b do -p macosx,macosx64,win32,linux,linux32 -u all -t none", "repository": "try", "revision": "12e26c5d0d57"}], "revision_hash": "28d50025d238a4410a844f11d6cec95cf2abf214", "jobs": [{"build_platform": {"platform": "OS X 10.6", "os_name": "mac", "architecture": "x86_64", "vm": false}, "submit_timestamp": 1366331470, "start_timestamp": 1366356690.0, "name": "reftest", "option_collection": {"debug": true}, "log_references": [{"url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/try-builds/mduan@mozilla.com-5cfd4cc28845/try-macosx64-debug/try_lion-debug_test-reftest-bm75-tests1-macosx-build37.txt.gz", "name": "unittest"}], "who": "mduan@mozilla.com", "reason": "scheduler", "artifact": {}, "machine_platform": {"platform": "OS X 10.6", "os_name": "mac", "architecture": "x86_64", "vm": false}, "machine": "talos-r4-lion-010", "state": "TODO", "result": 0, "job_guid": "8cc1cfe5e467d45f220bc7e63781a375b5582d48", "product_name": "firefox", "end_timestamp": "1366334018"}]} +{"sources": [{"commit_timestamp": 1366331469, "push_timestamp": 1366331469, "comments": "try: -b do -p macosx,macosx64,win32,linux,linux32 -u all -t none", "repository": "try", "revision": "12e26c5d0d57"}], "revision_hash": "28d50025d238a4410a844f11d6cec95cf2abf214", "job": {"build_platform": {"platform": "OS X 10.6", "os_name": "mac", "architecture": "x86_64", "vm": false}, "submit_timestamp": 1366331470, "start_timestamp": 1366356690.0, "name": "reftest", "option_collection": {"debug": true}, "log_references": [{"url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/try-builds/mduan@mozilla.com-5cfd4cc28845/try-macosx64-debug/try_lion-debug_test-reftest-bm75-tests1-macosx-build37.txt.gz", "name": "unittest"}], "who": "mduan@mozilla.com", "reason": "scheduler", "artifact": {}, "machine_platform": {"platform": "OS X 10.6", "os_name": "mac", "architecture": "x86_64", "vm": false}, "machine": "talos-r4-lion-010", "state": "TODO", "result": 0, "job_guid": "8cc1cfe5e467d45f220bc7e63781a375b5582d48", "product_name": "firefox", "end_timestamp": "1366334018"}} {"sources": [{"commit_timestamp": 1366332511, "push_timestamp": 1366332511, "comments": "Bug 856351 - Paris binding for DragEvent, r=peterv", "repository": "mozilla-inbound", "revision": "4b5381804b7c"}], "revision_hash": "97932524871cb0306f587baab1f4200dae27dffc", "jobs": [{"build_platform": {"platform": "2.2", "os_name": "android", "architecture": "ARMv7", "vm": false}, "submit_timestamp": 1366332549, "start_timestamp": 1366357753.0, "name": "jsreftest-1", "option_collection": {"opt": true}, "log_references": [{"url": "http://ftp.mozilla.org/pub/mozilla.org/mobile/tinderbox-builds/mozilla-inbound-android-noion/1366326906/mozilla-inbound_tegra_android-noion_test-jsreftest-1-bm22-tests1-tegra-build2402.txt.gz", "name": "unittest"}], "who": "sendchange-unittest", "reason": "scheduler", "artifact": {}, "machine_platform": {"platform": "2.2", "os_name": "android", "architecture": "ARMv7", "vm": false}, "machine": "tegra-206", "state": "TODO", "result": 0, "job_guid": "6ff0cc5fb09aed3ef2c1beaa408a478a83ba2f78", "product_name": "mobile", "end_timestamp": "1366334021"}]} {"sources": [{"commit_timestamp": 1366332167, "push_timestamp": 1366332167, "comments": "Bug 856351 - Paris binding for DragEvent, r=peterv", "repository": "mozilla-inbound", "revision": "4b5381804b7c"}], "revision_hash": "97932524871cb0306f587baab1f4200dae27dffc", "jobs": [{"build_platform": {"platform": "2.2", "os_name": "android", "architecture": "ARMv7", "vm": false}, "submit_timestamp": 1366332191, "start_timestamp": 1366357394.0, "name": "plain-reftest-4", "option_collection": {"opt": true}, "log_references": [{"url": "http://ftp.mozilla.org/pub/mozilla.org/mobile/tinderbox-builds/mozilla-inbound-android-armv6/1366326906/mozilla-inbound_tegra_android-armv6_test-plain-reftest-4-bm19-tests1-tegra-build978.txt.gz", "name": "unittest"}], "who": "sendchange-unittest", "reason": "scheduler", "artifact": {}, "machine_platform": {"platform": "2.2", "os_name": "android", "architecture": "ARMv7", "vm": false}, "machine": "tegra-228", "state": "TODO", "result": 0, "job_guid": "a3c4ee0b561dd5a1c444b41424612d9bd6d4cd8d", "product_name": "mobile", "end_timestamp": "1366334024"}]} {"sources": [{"commit_timestamp": 1366332895, "push_timestamp": 1366332895, "comments": "Bug 861317 - nsChildView::WillPaint terribly inefficient when drawing to titlebar. r=bgirard", "repository": "mozilla-inbound", "revision": "e75ff759c5c1"}], "revision_hash": "57c1fbabe544656b4edb96ad2c5678fda8c5e46d", "jobs": [{"build_platform": {"platform": "WINNT 6.3", "os_name": "win", "architecture": "x86_64", "vm": false}, "submit_timestamp": 1366332906, "start_timestamp": 1366358108.0, "name": "chromez", "option_collection": {"opt": true}, "log_references": [{"url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/tinderbox-builds/mozilla-inbound-win32/1366328526/mozilla-inbound_win8_test-chromez-bm15-tests1-windows-build188.txt.gz", "name": "talos"}], "who": "sendchange", "reason": "scheduler", "artifact": {}, "machine_platform": {"platform": "WINNT 6.3", "os_name": "win", "architecture": "x86_64", "vm": false}, "machine": "t-w864-ix-079", "state": "TODO", "result": 0, "job_guid": "cd6e1a5dc81cc0279acf4a097ab9745ff45d708a", "product_name": "firefox", "end_timestamp": "1366334026"}]} diff --git a/treeherder/model/derived/jobs.py b/treeherder/model/derived/jobs.py index 7b6c61400..2e1aae643 100644 --- a/treeherder/model/derived/jobs.py +++ b/treeherder/model/derived/jobs.py @@ -185,7 +185,7 @@ class JobsModel(TreeherderModelBase): # set Job data rdm = self.refdata_model - job = data["jobs"] + job = data["job"] build_platform_id = rdm.get_or_create_build_platform( job["build_platform"]["os_name"], @@ -258,7 +258,7 @@ class JobsModel(TreeherderModelBase): log_ref["url"] ) - except KeyError: + except (KeyError, JobDataError): # it is ok to have an empty or missing artifact pass