From 589817d6b09545f8bbfab922a7153087e0612463 Mon Sep 17 00:00:00 2001 From: Ed Morley Date: Thu, 20 Apr 2017 02:19:10 +0100 Subject: [PATCH] Bug 1357970 - Remove the artifact REST endpoint Artifacts no longer exist (they've been replaced by more specific types like "jobdetails"), and so fetching from this endpoint has been disabled for some time already. For data submission, we still call them artifacts (and sort their type after submission), however all artifacts are currently submitted at the same time as the job, so this endpoint is unused. --- docs/submitting_data.rst | 74 +-- tests/client/data/artifact_data.json | 667 ------------------------- tests/client/test_treeherder_client.py | 127 +---- tests/webapp/api/test_artifact_api.py | 36 -- treeherder/client/thclient/client.py | 68 --- treeherder/webapp/api/artifact.py | 29 -- treeherder/webapp/api/urls.py | 9 +- 7 files changed, 4 insertions(+), 1006 deletions(-) delete mode 100644 tests/client/data/artifact_data.json delete mode 100644 tests/webapp/api/test_artifact_api.py delete mode 100644 treeherder/webapp/api/artifact.py diff --git a/docs/submitting_data.rst b/docs/submitting_data.rst index 78858e1c8..9e73064a5 100644 --- a/docs/submitting_data.rst +++ b/docs/submitting_data.rst @@ -262,25 +262,6 @@ characters at most. A job collection has the following data structure. see :ref:`custom-log-name` for more info. -Artifact Collections -^^^^^^^^^^^^^^^^^^^^ - -Artifact collections contain arbitrary data associated with a job. This is -usually a json blob of structured data produced by the build system during the -job execution. - -.. code-block:: python - - [ - { - 'type': 'json', - 'name': 'my-artifact-name', - # blob can be any kind of structured data - 'blob': { 'stuff': [1, 2, 3, 4, 5] }, - 'job_guid': 'd22c74d4aa6d2a1dcba96d95dccbd5fdca70cf33' - } - ] - Usage ^^^^^ @@ -392,31 +373,6 @@ structures to send, do something like this: client = TreeherderClient(client_id='hawk_id', secret='hawk_secret') client.post_collection('mozilla-central', tjc) -If you want to use `TreeherderArtifactCollection` to build up the job -artifacts data structures to send, do something like this: - -.. code-block:: python - - from thclient import (TreeherderClient, TreeherderClientError, - TreeherderArtifactCollection) - - tac = TreeherderArtifactCollection() - - for data in dataset: - - ta = tac.get_artifact() - - ta.add_blob( data['blob'] ) - ta.add_name( data['name'] ) - ta.add_type( data['type'] ) - ta.add_job_guid( data['job_guid'] ) - - tac.add(ta) - - # Send the collection to treeherder - client = TreeherderClient(client_id='hawk_id', secret='hawk_secret') - client.post_collection('mozilla-central', tac) - If you don't want to use `TreeherderResultCollection` or `TreeherderJobCollection` to build up the data structure to send, build the data structures directly and add them to the collection. @@ -455,27 +411,6 @@ data structures directly and add them to the collection. client = TreeherderClient(client_id='hawk_id', secret='hawk_secret') client.post_collection('mozilla-central', tjc) -In the same way, if you don't want to use `TreeherderArtifactCollection` to -build up the data structure to send, build the data structures directly and -add them to the collection. - -.. code-block:: python - - from thclient import TreeherderClient, TreeherderArtifactCollection - - tac = TreeherderArtifactCollection() - - for artifact in artifact_data: - ta = tac.get_artifact(artifact) - - # Add any additional data to ta.data here - - # add artifact to collection - tac.add(ta) - - client = TreeherderClient(client_id='hawk_id', secret='hawk_secret') - client.post_collection('mozilla-central', tac) - Job artifacts format ^^^^^^^^^^^^^^^^^^^^ @@ -605,15 +540,10 @@ Via the ``/jobs`` endpoint: 3. Submit a Log URL with ``parse_status`` of "parsed", with ``text_log_summary`` and ``Bug suggestions`` artifacts * Will generate nothing - -ArtifactCollections -~~~~~~~~~~~~~~~~~~~ -Via the ``/artifact`` endpoint: - -1. Submit a ``text_log_summary`` artifact +4. Submit a ``text_log_summary`` artifact * Will generate a ``Bug suggestions`` artifact if it does not already exist for that job. -2. Submit ``text_log_summary`` and ``Bug suggestions`` artifacts +5. Submit ``text_log_summary`` and ``Bug suggestions`` artifacts * Will generate nothing * This is *Treeherder's* current internal log parser workflow diff --git a/tests/client/data/artifact_data.json b/tests/client/data/artifact_data.json deleted file mode 100644 index 6b312fe98..000000000 --- a/tests/client/data/artifact_data.json +++ /dev/null @@ -1,667 +0,0 @@ -[ - { - "blob": { - "tinderbox_printlines": ["mozharness_revlink: https://hg.mozilla.org/build/mozharness/rev/26cd65055bac", "reftest-jsreftest
57438/0/1209\r"], - "logurl": "http://ftp.mozilla.org/pub/mozilla.org/firefox/try-builds/jwatt@jwatt.org-ecedbb0a251b/try-win32/try_xp-ix_test-jsreftest-bm111-tests1-windows-build153.txt.gz" - }, - "type": "json", - "name": "Job Info", - "job_guid": 32698 - }, - { - "blob": { - "step_data": { - "steps": [ - { - "errors": [], - "name": "set props: master", - "started": "2014-03-07 07:12:03.636406", - "started_linenumber": 8, - "finished_linenumber": 10, - "finished": "2014-03-07 07:12:03.636818" - }, - { - "errors": [], - "name": "set props: basedir", - "started": "2014-03-07 07:12:03.637105", - "started_linenumber": 12, - "finished_linenumber": 55, - "finished": "2014-03-07 07:12:03.879738" - }, - { - "errors": [], - "name": "downloading to buildprops.json", - "started": "2014-03-07 07:12:03.880087", - "started_linenumber": 57, - "finished_linenumber": 58, - "finished": "2014-03-07 07:12:03.924320" - }, - { - "errors": [], - "name": "'rm -rf ...'", - "started": "2014-03-07 07:12:03.924713", - "started_linenumber": 60, - "finished_linenumber": 101, - "finished": "2014-03-07 07:12:04.181933" - }, - { - "errors": [], - "name": "'rm -rf ...'", - "started": "2014-03-07 07:12:04.182263", - "started_linenumber": 103, - "finished_linenumber": 144, - "finished": "2014-03-07 07:12:08.199403" - }, - { - "errors": [], - "name": "'c:\\mozilla-build\\hg\\hg clone ...'", - "started": "2014-03-07 07:12:08.199729", - "started_linenumber": 146, - "finished_linenumber": 194, - "finished": "2014-03-07 07:12:12.978266" - }, - { - "errors": [], - "name": "'c:\\mozilla-build\\hg\\hg update ...'", - "started": "2014-03-07 07:12:12.978609", - "started_linenumber": 196, - "finished_linenumber": 238, - "finished": "2014-03-07 07:12:13.389483" - }, - { - "errors": [], - "name": "set props: script_repo_revision", - "started": "2014-03-07 07:12:13.389785", - "started_linenumber": 240, - "finished_linenumber": 283, - "finished": "2014-03-07 07:12:13.658294" - }, - { - "errors": [], - "name": "downloading to oauth.txt", - "started": "2014-03-07 07:12:13.658595", - "started_linenumber": 285, - "finished_linenumber": 286, - "finished": "2014-03-07 07:12:13.670954" - }, - { - "errors": [], - "name": "tinderboxprint_script_revlink", - "started": "2014-03-07 07:12:13.671280", - "started_linenumber": 288, - "finished_linenumber": 290, - "finished": "2014-03-07 07:12:13.671644" - }, - { - "errors": [], - "name": "'c:/mozilla-build/python27/python -u ...'", - "started": "2014-03-07 07:12:13.671901", - "started_linenumber": 292, - "finished_linenumber": 97498, - "finished": "2014-03-07 07:22:58.578709" - }, - { - "errors": [], - "name": "set props: build_url", - "started": "2014-03-07 07:22:58.584633", - "started_linenumber": 97500, - "finished_linenumber": 97543, - "finished": "2014-03-07 07:22:58.850768" - }, - { - "errors": [], - "name": "'rm -f ...'", - "started": "2014-03-07 07:22:58.851093", - "started_linenumber": 97545, - "finished_linenumber": 97586, - "finished": "2014-03-07 07:22:59.078797" - }, - { - "errors": [], - "name": "reboot slave lost", - "started": "2014-03-07 07:22:59.079128", - "started_linenumber": 97588, - "finished_linenumber": 97590, - "finished": "2014-03-07 07:23:00.519294" - } - ] - }, - "logurl": "http://ftp.mozilla.org/pub/mozilla.org/firefox/try-builds/jwatt@jwatt.org-ecedbb0a251b/try-win32/try_xp-ix_test-jsreftest-bm111-tests1-windows-build153.txt.gz" - }, - "type": "json", - "name": "Structured Log", - "job_guid": 32698 - }, - { - "blob": { - "tinderbox_printlines": ["mozharness_revlink: https://hg.mozilla.org/build/mozharness/rev/26cd65055bac", "TalosResult: {\"datazilla\": {\"a11yr\": {\"url\": \"https://datazilla.mozilla.org?x86_64=false&product=Firefox&repository=Try-Non-PGO&os_version=Ubuntu%2012.04&stop=1394205845&project=talos&start=1393601045&graph_search=3125296c850d&os=linux&test=a11yr\"}, \"tpaint\": {\"url\": \"https://datazilla.mozilla.org?x86_64=false&product=Firefox&repository=Try-Non-PGO&os_version=Ubuntu%2012.04&stop=1394205845&project=talos&start=1393601045&graph_search=3125296c850d&os=linux&test=a11yr&test=ts_paint&test=tpaint\"}, \"ts_paint\": {\"url\": \"https://datazilla.mozilla.org?x86_64=false&product=Firefox&repository=Try-Non-PGO&os_version=Ubuntu%2012.04&stop=1394205845&project=talos&start=1393601045&graph_search=3125296c850d&os=linux&test=a11yr&test=ts_paint\"}}, \"graphserver\": {\"a11yr_paint\": {\"url\": \"http://graphs.mozilla.org/graph.html#tests=[[223,113,33]]\", \"result\": \"351.00\"}, \"tpaint\": {\"url\": \"http://graphs.mozilla.org/graph.html#tests=[[82,113,33]]\", \"result\": \"182.76\"}, \"ts_paint\": {\"url\": \"http://graphs.mozilla.org/graph.html#tests=[[83,113,33]]\", \"result\": \"857.16\"}}}"], - "logurl": "http://ftp.mozilla.org/pub/mozilla.org/firefox/try-builds/chrislord.net@gmail.com-3125296c850d/try-linux/try_ubuntu32_hw_test-other-bm103-tests1-linux-build150.txt.gz" - }, - "type": "json", - "name": "Job Info", - "job_guid": 33028 - }, - { - "blob": { - "step_data": { - "steps": [ - { - "errors": [], - "name": "set props: master", - "started": "2014-03-07 07:18:22.729550", - "started_linenumber": 8, - "finished_linenumber": 10, - "finished": "2014-03-07 07:18:22.729923" - }, - { - "errors": [], - "name": "set props: basedir", - "started": "2014-03-07 07:18:22.730379", - "started_linenumber": 12, - "finished_linenumber": 60, - "finished": "2014-03-07 07:18:22.774796" - }, - { - "errors": [], - "name": "downloading to buildprops.json", - "started": "2014-03-07 07:18:22.775105", - "started_linenumber": 62, - "finished_linenumber": 63, - "finished": "2014-03-07 07:18:22.819873" - }, - { - "errors": [], - "name": "'rm -rf ...'", - "started": "2014-03-07 07:18:22.820284", - "started_linenumber": 65, - "finished_linenumber": 111, - "finished": "2014-03-07 07:18:22.858509" - }, - { - "errors": [], - "name": "'rm -rf ...'", - "started": "2014-03-07 07:18:22.858795", - "started_linenumber": 113, - "finished_linenumber": 159, - "finished": "2014-03-07 07:18:22.907592" - }, - { - "errors": [], - "name": "'hg clone ...'", - "started": "2014-03-07 07:18:22.907870", - "started_linenumber": 161, - "finished_linenumber": 214, - "finished": "2014-03-07 07:18:26.841475" - }, - { - "errors": [], - "name": "'hg update ...'", - "started": "2014-03-07 07:18:26.841785", - "started_linenumber": 216, - "finished_linenumber": 263, - "finished": "2014-03-07 07:18:27.404940" - }, - { - "errors": [], - "name": "set props: script_repo_revision", - "started": "2014-03-07 07:18:27.405297", - "started_linenumber": 265, - "finished_linenumber": 313, - "finished": "2014-03-07 07:18:27.750008" - }, - { - "errors": [], - "name": "downloading to oauth.txt", - "started": "2014-03-07 07:18:27.753553", - "started_linenumber": 315, - "finished_linenumber": 316, - "finished": "2014-03-07 07:18:27.908032" - }, - { - "errors": [], - "name": "tinderboxprint_script_revlink", - "started": "2014-03-07 07:18:27.908301", - "started_linenumber": 318, - "finished_linenumber": 320, - "finished": "2014-03-07 07:18:27.908645" - }, - { - "errors": [], - "name": "'/tools/buildbot/bin/python scripts/scripts/talos_script.py ...'", - "started": "2014-03-07 07:18:27.908937", - "started_linenumber": 322, - "finished_linenumber": 1739, - "finished": "2014-03-07 07:24:06.327614" - }, - { - "errors": [], - "name": "set props: build_url", - "started": "2014-03-07 07:24:06.332695", - "started_linenumber": 1741, - "finished_linenumber": 1789, - "finished": "2014-03-07 07:24:06.361854" - }, - { - "errors": [], - "name": "'rm -f ...'", - "started": "2014-03-07 07:24:06.362241", - "started_linenumber": 1791, - "finished_linenumber": 1837, - "finished": "2014-03-07 07:24:06.380120" - }, - { - "errors": [], - "name": "reboot slave lost", - "started": "2014-03-07 07:24:06.380445", - "started_linenumber": 1839, - "finished_linenumber": 1841, - "finished": "2014-03-07 07:24:10.781191" - } - ] - }, - "logurl": "http://ftp.mozilla.org/pub/mozilla.org/firefox/try-builds/chrislord.net@gmail.com-3125296c850d/try-linux/try_ubuntu32_hw_test-other-bm103-tests1-linux-build150.txt.gz" - }, - "type": "json", - "name": "Structured Log", - "job_guid": 33028 - }, - { - "blob": { - "tinderbox_printlines": ["mozharness_revlink: https://hg.mozilla.org/build/mozharness/rev/26cd65055bac", "mochitest-plain2
214295/0/24256"], - "logurl": "http://ftp.mozilla.org/pub/mozilla.org/firefox/try-builds/chrislord.net@gmail.com-3125296c850d/try-linux64/try_ubuntu64_vm_test-mochitest-2-bm68-tests1-linux-build1984.txt.gz" - }, - "type": "json", - "name": "Job Info", - "job_guid": 32956 - }, - { - "blob": { - "step_data": { - "steps": [ - { - "errors": [], - "name": "set props: master", - "started": "2014-03-07 07:14:41.792299", - "started_linenumber": 8, - "finished_linenumber": 10, - "finished": "2014-03-07 07:14:41.792958" - }, - { - "errors": [], - "name": "set props: basedir", - "started": "2014-03-07 07:14:41.793432", - "started_linenumber": 12, - "finished_linenumber": 56, - "finished": "2014-03-07 07:14:41.858335" - }, - { - "errors": [], - "name": "downloading to buildprops.json", - "started": "2014-03-07 07:14:41.858813", - "started_linenumber": 58, - "finished_linenumber": 59, - "finished": "2014-03-07 07:14:41.893872" - }, - { - "errors": [], - "name": "'rm -rf ...'", - "started": "2014-03-07 07:14:41.894339", - "started_linenumber": 61, - "finished_linenumber": 103, - "finished": "2014-03-07 07:14:41.990805" - }, - { - "errors": [], - "name": "'rm -rf ...'", - "started": "2014-03-07 07:14:41.991277", - "started_linenumber": 105, - "finished_linenumber": 147, - "finished": "2014-03-07 07:14:42.387667" - }, - { - "errors": [], - "name": "'hg clone ...'", - "started": "2014-03-07 07:14:42.388135", - "started_linenumber": 149, - "finished_linenumber": 198, - "finished": "2014-03-07 07:14:46.490504" - }, - { - "errors": [], - "name": "'hg update ...'", - "started": "2014-03-07 07:14:46.491031", - "started_linenumber": 200, - "finished_linenumber": 243, - "finished": "2014-03-07 07:14:46.959315" - }, - { - "errors": [], - "name": "set props: script_repo_revision", - "started": "2014-03-07 07:14:46.959819", - "started_linenumber": 245, - "finished_linenumber": 289, - "finished": "2014-03-07 07:14:47.217313" - }, - { - "errors": [], - "name": "downloading to oauth.txt", - "started": "2014-03-07 07:14:47.217835", - "started_linenumber": 291, - "finished_linenumber": 292, - "finished": "2014-03-07 07:14:47.238172" - }, - { - "errors": [], - "name": "tinderboxprint_script_revlink", - "started": "2014-03-07 07:14:47.238666", - "started_linenumber": 294, - "finished_linenumber": 296, - "finished": "2014-03-07 07:14:47.239335" - }, - { - "errors": [], - "name": "'/tools/buildbot/bin/python scripts/scripts/desktop_unittest.py ...'", - "started": "2014-03-07 07:14:47.239846", - "started_linenumber": 298, - "finished_linenumber": 10126, - "finished": "2014-03-07 07:24:34.027012" - }, - { - "errors": [], - "name": "set props: build_url", - "started": "2014-03-07 07:24:34.028119", - "started_linenumber": 10128, - "finished_linenumber": 10172, - "finished": "2014-03-07 07:24:34.086304" - }, - { - "errors": [], - "name": "'rm -f ...'", - "started": "2014-03-07 07:24:34.086806", - "started_linenumber": 10174, - "finished_linenumber": 10216, - "finished": "2014-03-07 07:24:34.129472" - }, - { - "errors": [], - "name": "reboot slave lost", - "started": "2014-03-07 07:24:34.129930", - "started_linenumber": 10218, - "finished_linenumber": 10220, - "finished": "2014-03-07 07:24:38.385365" - } - ] - }, - "logurl": "http://ftp.mozilla.org/pub/mozilla.org/firefox/try-builds/chrislord.net@gmail.com-3125296c850d/try-linux64/try_ubuntu64_vm_test-mochitest-2-bm68-tests1-linux-build1984.txt.gz" - }, - "type": "json", - "name": "Structured Log", - "job_guid": 32956 - }, - { - "blob": { - "tinderbox_printlines": ["mozharness_revlink: https://hg.mozilla.org/build/mozharness/rev/26cd65055bac", "reftest-crashtest
2616/0/14"], - "logurl": "http://ftp.mozilla.org/pub/mozilla.org/firefox/try-builds/chrislord.net@gmail.com-3125296c850d/try-linux64/try_ubuntu64_vm_test-crashtest-bm52-tests1-linux-build1710.txt.gz" - }, - "type": "json", - "name": "Job Info", - "job_guid": 32975 - }, - { - "blob": { - "step_data": { - "steps": [ - { - "errors": [], - "name": "set props: master", - "started": "2014-03-07 07:14:34.298831", - "started_linenumber": 8, - "finished_linenumber": 10, - "finished": "2014-03-07 07:14:34.299526" - }, - { - "errors": [], - "name": "set props: basedir", - "started": "2014-03-07 07:14:34.299969", - "started_linenumber": 12, - "finished_linenumber": 56, - "finished": "2014-03-07 07:14:36.493319" - }, - { - "errors": [], - "name": "downloading to buildprops.json", - "started": "2014-03-07 07:14:36.493812", - "started_linenumber": 58, - "finished_linenumber": 59, - "finished": "2014-03-07 07:14:36.842727" - }, - { - "errors": [], - "name": "'rm -rf ...'", - "started": "2014-03-07 07:14:36.843133", - "started_linenumber": 61, - "finished_linenumber": 103, - "finished": "2014-03-07 07:14:36.917108" - }, - { - "errors": [], - "name": "'rm -rf ...'", - "started": "2014-03-07 07:14:36.917553", - "started_linenumber": 105, - "finished_linenumber": 147, - "finished": "2014-03-07 07:14:38.049526" - }, - { - "errors": [], - "name": "'hg clone ...'", - "started": "2014-03-07 07:14:38.050029", - "started_linenumber": 149, - "finished_linenumber": 198, - "finished": "2014-03-07 07:14:53.802561" - }, - { - "errors": [], - "name": "'hg update ...'", - "started": "2014-03-07 07:14:53.803111", - "started_linenumber": 200, - "finished_linenumber": 243, - "finished": "2014-03-07 07:14:54.140737" - }, - { - "errors": [], - "name": "set props: script_repo_revision", - "started": "2014-03-07 07:14:54.141318", - "started_linenumber": 245, - "finished_linenumber": 289, - "finished": "2014-03-07 07:14:54.330501" - }, - { - "errors": [], - "name": "downloading to oauth.txt", - "started": "2014-03-07 07:14:54.330976", - "started_linenumber": 291, - "finished_linenumber": 292, - "finished": "2014-03-07 07:14:54.354151" - }, - { - "errors": [], - "name": "tinderboxprint_script_revlink", - "started": "2014-03-07 07:14:54.354709", - "started_linenumber": 294, - "finished_linenumber": 296, - "finished": "2014-03-07 07:14:54.355354" - }, - { - "errors": [], - "name": "'/tools/buildbot/bin/python scripts/scripts/desktop_unittest.py ...'", - "started": "2014-03-07 07:14:54.355786", - "started_linenumber": 298, - "finished_linenumber": 17699, - "finished": "2014-03-07 07:22:43.052819" - }, - { - "errors": [], - "name": "set props: build_url", - "started": "2014-03-07 07:22:43.085835", - "started_linenumber": 17701, - "finished_linenumber": 17745, - "finished": "2014-03-07 07:22:43.401531" - }, - { - "errors": [], - "name": "'rm -f ...'", - "started": "2014-03-07 07:22:43.402049", - "started_linenumber": 17747, - "finished_linenumber": 17789, - "finished": "2014-03-07 07:22:43.451869" - }, - { - "errors": [], - "name": "reboot slave lost", - "started": "2014-03-07 07:22:43.452327", - "started_linenumber": 17791, - "finished_linenumber": 17793, - "finished": "2014-03-07 07:22:48.221966" - } - ] - }, - "logurl": "http://ftp.mozilla.org/pub/mozilla.org/firefox/try-builds/chrislord.net@gmail.com-3125296c850d/try-linux64/try_ubuntu64_vm_test-crashtest-bm52-tests1-linux-build1710.txt.gz" - }, - "type": "json", - "name": "Structured Log", - "job_guid": 32975 - }, - { - "blob": { - "tinderbox_printlines": ["mozharness_revlink: https://hg.mozilla.org/build/mozharness/rev/26cd65055bac", "mochitest-plain3
39682/0/176"], - "logurl": "http://ftp.mozilla.org/pub/mozilla.org/firefox/try-builds/chrislord.net@gmail.com-3125296c850d/try-linux64/try_ubuntu64_vm_test-mochitest-3-bm52-tests1-linux-build1885.txt.gz" - }, - "type": "json", - "name": "Job Info", - "job_guid": 32968 - }, - { - "blob": { - "step_data": { - "steps": [ - { - "errors": [], - "name": "set props: master", - "started": "2014-03-07 07:14:30.412431", - "started_linenumber": 8, - "finished_linenumber": 10, - "finished": "2014-03-07 07:14:30.413108" - }, - { - "errors": [], - "name": "set props: basedir", - "started": "2014-03-07 07:14:30.413529", - "started_linenumber": 12, - "finished_linenumber": 56, - "finished": "2014-03-07 07:14:30.758181" - }, - { - "errors": [], - "name": "downloading to buildprops.json", - "started": "2014-03-07 07:14:30.758641", - "started_linenumber": 58, - "finished_linenumber": 59, - "finished": "2014-03-07 07:14:30.805123" - }, - { - "errors": [], - "name": "'rm -rf ...'", - "started": "2014-03-07 07:14:30.805562", - "started_linenumber": 61, - "finished_linenumber": 103, - "finished": "2014-03-07 07:14:30.859079" - }, - { - "errors": [], - "name": "'rm -rf ...'", - "started": "2014-03-07 07:14:30.859546", - "started_linenumber": 105, - "finished_linenumber": 147, - "finished": "2014-03-07 07:14:31.271884" - }, - { - "errors": [], - "name": "'hg clone ...'", - "started": "2014-03-07 07:14:31.272719", - "started_linenumber": 149, - "finished_linenumber": 198, - "finished": "2014-03-07 07:14:36.816646" - }, - { - "errors": [], - "name": "'hg update ...'", - "started": "2014-03-07 07:14:36.817194", - "started_linenumber": 200, - "finished_linenumber": 243, - "finished": "2014-03-07 07:14:38.062517" - }, - { - "errors": [], - "name": "set props: script_repo_revision", - "started": "2014-03-07 07:14:38.062993", - "started_linenumber": 245, - "finished_linenumber": 289, - "finished": "2014-03-07 07:14:38.241590" - }, - { - "errors": [], - "name": "downloading to oauth.txt", - "started": "2014-03-07 07:14:38.242080", - "started_linenumber": 291, - "finished_linenumber": 292, - "finished": "2014-03-07 07:14:38.267092" - }, - { - "errors": [], - "name": "tinderboxprint_script_revlink", - "started": "2014-03-07 07:14:38.267614", - "started_linenumber": 294, - "finished_linenumber": 296, - "finished": "2014-03-07 07:14:38.268277" - }, - { - "errors": [], - "name": "'/tools/buildbot/bin/python scripts/scripts/desktop_unittest.py ...'", - "started": "2014-03-07 07:14:38.268708", - "started_linenumber": 298, - "finished_linenumber": 32014, - "finished": "2014-03-07 07:24:06.398324" - }, - { - "errors": [], - "name": "set props: build_url", - "started": "2014-03-07 07:24:06.403049", - "started_linenumber": 32016, - "finished_linenumber": 32060, - "finished": "2014-03-07 07:24:06.464080" - }, - { - "errors": [], - "name": "'rm -f ...'", - "started": "2014-03-07 07:24:06.464585", - "started_linenumber": 32062, - "finished_linenumber": 32104, - "finished": "2014-03-07 07:24:06.507310" - }, - { - "errors": [], - "name": "reboot slave lost", - "started": "2014-03-07 07:24:06.507796", - "started_linenumber": 32106, - "finished_linenumber": 32108, - "finished": "2014-03-07 07:24:11.035626" - } - ] - }, - "logurl": "http://ftp.mozilla.org/pub/mozilla.org/firefox/try-builds/chrislord.net@gmail.com-3125296c850d/try-linux64/try_ubuntu64_vm_test-mochitest-3-bm52-tests1-linux-build1885.txt.gz" - }, - "type": "json", - "name": "Structured Log", - "job_guid": 32968 - } -] diff --git a/tests/client/test_treeherder_client.py b/tests/client/test_treeherder_client.py index 384f1a6ab..b59e899aa 100644 --- a/tests/client/test_treeherder_client.py +++ b/tests/client/test_treeherder_client.py @@ -7,9 +7,7 @@ import unittest import responses from requests_hawk import HawkAuth -from treeherder.client.thclient import (TreeherderArtifact, - TreeherderArtifactCollection, - TreeherderClient, +from treeherder.client.thclient import (TreeherderClient, TreeherderClientError, TreeherderJob, TreeherderJobCollection, @@ -56,21 +54,6 @@ class DataSetup(unittest.TestCase): resultset['type'] = 'push' resultset['author'] = 'somebody@somewhere.com' - # Load sample artifact - - self.artifact_data = [] - - artifact_file = 'artifact_data.json' - - self.artifact_data_file_path = os.path.join( - os.path.dirname(__file__), - 'data', - artifact_file - ) - - with open(self.artifact_data_file_path) as f: - self.artifact_data = json.load(f) - def compare_structs(self, struct1, struct2): """Compare two dictionary structures""" @@ -176,88 +159,6 @@ class TreeherderResultSetCollectionTest(DataSetup, unittest.TestCase): self.assertTrue(len(self.resultset_data) == len(trc.data)) -class TreeherderArtifactTest(DataSetup, unittest.TestCase): - - def test_sample_data_validation(self): - """Confirm that the sample data validates""" - - for artifact in self.artifact_data: - - rs = TreeherderArtifact(artifact) - rs.validate() - - def test_artifact_sample_data(self): - """Test all add methods for building an artifact""" - - tac = TreeherderArtifactCollection() - - for artifact in self.artifact_data: - - ta = TreeherderArtifact() - - ta.add_blob(artifact['blob']) - ta.add_job_guid(artifact['job_guid']) - ta.add_name(artifact['name']) - ta.add_type(artifact['type']) - - self.compare_structs(ta.data, artifact) - - tac.add(ta) - - # confirm we get the same thing if we initialize from - # a resultset dict - ta_struct = TreeherderArtifact(artifact) - - self.compare_structs(ta_struct.data, artifact) - - -class TreeherderArtifactCollectionTest(DataSetup, unittest.TestCase): - - def test_artifact_collection(self): - """Confirm the collection matches the sample data""" - tac = TreeherderArtifactCollection() - - for artifact in self.artifact_data: - ta = TreeherderArtifact(artifact) - tac.add(ta) - - self.assertTrue(len(self.artifact_data) == len(tac.data)) - - def test_collection_chunking(self): - tac = TreeherderArtifactCollection() - - for artifact in self.artifact_data: - ta = TreeherderArtifact(artifact) - tac.add(ta) - - # reconstruct the chunks and make sure we have the same data - rebuilt_data = [] - chunk_num = 0 - for chunk in tac.get_chunks(3): - chunk_data = chunk.get_collection_data() - rebuilt_data.extend(chunk_data) - - chunk_num += 1 - # the last one should be the "remainder" in an uneven size - if chunk_num == 4: - assert len(chunk_data) == 1 - else: - assert len(chunk_data) == 3 - - assert rebuilt_data == tac.get_collection_data() - - def test_chunk_endpoint_base(self): - """Confirm the endpoint_base of the chunks is the same as the original""" - tac = TreeherderArtifactCollection() - - for artifact in self.artifact_data: - ta = TreeherderArtifact(artifact) - tac.add(ta) - - for chunk in tac.get_chunks(3): - assert tac.endpoint_base == chunk.endpoint_base - - class TreeherderJobCollectionTest(DataSetup, unittest.TestCase): def test_job_collection(self): @@ -439,32 +340,6 @@ class TreeherderClientTest(DataSetup, unittest.TestCase): client.post_collection('project', trc) - @responses.activate - def test_send_artifact_collection(self): - """Can add a artifact collections to a TreeherderRequest.""" - tac = TreeherderArtifactCollection() - - for artifact in self.artifact_data: - tac.add(tac.get_artifact(artifact)) - - client = TreeherderClient( - server_url='http://host', - client_id='client-abc', - secret='secret123', - ) - - def request_callback(request): - # Check that the expected content was POSTed. - posted_json = json.loads(request.body) - self.assertEqual(posted_json, tac.get_collection_data()) - return (200, {}, '{"message": "Artifacts stored successfully"}') - - url = client._get_endpoint_url(tac.endpoint_base, project='project') - responses.add_callback(responses.POST, url, match_querystring=True, - callback=request_callback, content_type='application/json') - - client.post_collection('project', tac) - def test_hawkauth_setup(self): """Test that HawkAuth is correctly set up from the `client_id` and `secret` params.""" client = TreeherderClient( diff --git a/tests/webapp/api/test_artifact_api.py b/tests/webapp/api/test_artifact_api.py deleted file mode 100644 index 146402fc9..000000000 --- a/tests/webapp/api/test_artifact_api.py +++ /dev/null @@ -1,36 +0,0 @@ -import json - -import pytest - -from treeherder.client.thclient import client -from treeherder.model.models import (TextLogError, - TextLogStep) - -xfail = pytest.mark.xfail - - -def test_artifact_create_text_log_summary(webapp, test_repository, test_job, - mock_post_json, sample_data): - """ - test submitting a text_log_summary artifact creates some text log summary objects - """ - tls = sample_data.text_log_summary - - # assert that we had no text log objects before this operation - assert not TextLogStep.objects.filter(job=test_job).exists() - - tac = client.TreeherderArtifactCollection() - ta = client.TreeherderArtifact({ - 'type': 'json', - 'name': 'text_log_summary', - 'blob': json.dumps(tls['blob']), - 'job_guid': test_job.guid - }) - tac.add(ta) - - cli = client.TreeherderClient(server_url='http://localhost') - cli.post_collection(test_repository.name, tac) - - # assert we generated some objects - assert TextLogStep.objects.filter(job=test_job).count() > 0 - assert TextLogError.objects.filter(step__job=test_job).count() > 0 diff --git a/treeherder/client/thclient/client.py b/treeherder/client/thclient/client.py index 15b2f23ee..bc8da5e9c 100644 --- a/treeherder/client/thclient/client.py +++ b/treeherder/client/thclient/client.py @@ -444,49 +444,6 @@ class TreeherderResultSet(TreeherderData, ValidatorMixin): return TreeherderRevision(data) -class TreeherderArtifact(TreeherderData, ValidatorMixin): - """ - Supports building a treeherder job artifact - """ - - def __init__(self, data={}): - - super(TreeherderArtifact, self).__init__(data) - - # Provide minimal json structure validation - self.required_properties = { - 'blob': {'cb': self.validate_existence}, - 'type': {'cb': self.validate_existence}, - 'name': {'cb': self.validate_existence}, - 'job_guid': {'cb': self.validate_existence} - } - - def init_data(self): - - self.data = { - # Stored in project_jobs_1.artifact.blob - 'blob': '', - # Stored in project_jobs_1.artifact.type - 'type': '', - # Stored in project_jobs_1.artifact.name - 'name': '', - # Stored in project_jobs_1.artifact.job_guid - 'job_guid': None - } - - def add_blob(self, blob): - self.data['blob'] = blob - - def add_type(self, type): - self.data['type'] = type - - def add_name(self, name): - self.data['name'] = name - - def add_job_guid(self, job_guid): - self.data['job_guid'] = job_guid - - class TreeherderCollection(object): """ Base class for treeherder data collections @@ -578,20 +535,6 @@ class TreeherderResultSetCollection(TreeherderCollection): return TreeherderResultSet(data) -class TreeherderArtifactCollection(TreeherderCollection): - """ - Collection of job artifacts - """ - - def __init__(self, data=[]): - - super(TreeherderArtifactCollection, self).__init__('artifact', data) - - def get_artifact(self, data={}): - - return TreeherderArtifact(data) - - class TreeherderClient(object): """ Treeherder client class @@ -607,7 +550,6 @@ class TreeherderClient(object): JOBS_ENDPOINT = 'jobs' JOB_DETAIL_ENDPOINT = 'jobdetail' JOB_LOG_URL_ENDPOINT = 'job-log-url' - ARTIFACTS_ENDPOINT = 'artifact' OPTION_COLLECTION_HASH_ENDPOINT = 'optioncollectionhash' REPOSITORY_ENDPOINT = 'repository' JOBGROUP_ENDPOINT = 'jobgroup' @@ -881,16 +823,6 @@ class TreeherderClient(object): return self._get_json(self.JOB_LOG_URL_ENDPOINT, project, **params) - def get_artifacts(self, project, **params): - """ - Gets artifact list from project, filtered by parameters - - :param project: project (repository name) to query for - :param params: keyword arguments to filter results - """ - response = self._get_json(self.ARTIFACTS_ENDPOINT, project, **params) - return response - def post_collection(self, project, collection_inst): """ Sends a treeherder collection to the server diff --git a/treeherder/webapp/api/artifact.py b/treeherder/webapp/api/artifact.py deleted file mode 100644 index 14e12a671..000000000 --- a/treeherder/webapp/api/artifact.py +++ /dev/null @@ -1,29 +0,0 @@ -from rest_framework import viewsets -from rest_framework.response import Response -from rest_framework.status import HTTP_405_METHOD_NOT_ALLOWED - -from treeherder.etl.artifact import (serialize_artifact_json_blobs, - store_job_artifacts) -from treeherder.webapp.api import permissions - - -class ArtifactViewSet(viewsets.ViewSet): - permission_classes = (permissions.HasHawkPermissionsOrReadOnly,) - - """ - This viewset is responsible for the artifact endpoint. - """ - def retrieve(self, request, project, pk=None): - return Response("retrieving job artifacts is no longer supported", - status=HTTP_405_METHOD_NOT_ALLOWED) - - def list(self, request, project): - return Response("retrieving job artifacts is no longer supported", - status=HTTP_405_METHOD_NOT_ALLOWED) - - def create(self, request, project): - serialized_artifacts = serialize_artifact_json_blobs( - request.data) - store_job_artifacts(serialized_artifacts) - - return Response({'message': 'Artifacts stored successfully'}) diff --git a/treeherder/webapp/api/urls.py b/treeherder/webapp/api/urls.py index e9170158a..e53f9d7db 100644 --- a/treeherder/webapp/api/urls.py +++ b/treeherder/webapp/api/urls.py @@ -4,8 +4,7 @@ from django.conf.urls import (include, url) from rest_framework import routers -from treeherder.webapp.api import (artifact, - auth, +from treeherder.webapp.api import (auth, bug, bugzilla, classifiedfailure, @@ -56,12 +55,6 @@ project_bound_router.register( base_name='resultset', ) -project_bound_router.register( - r'artifact', - artifact.ArtifactViewSet, - base_name='artifact', -) - project_bound_router.register( r'note', note.NoteViewSet,