Bug 1248710 - Add a User Agent to fetchlogs.py's requests to Treeherder

Also removes some unused imports and whitespace flake8 warnings.

testing/web-platform/update/fetchlogs.py

@@ -3,9 +3,6 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import argparse
-import cStringIO
-import gzip
-import json
 import os
 import requests
 import urlparse
@@ -21,6 +18,7 @@ up to mach or similar"""
 # Interpretation of the "job" list from
 # https://github.com/mozilla/treeherder-service/blob/master/treeherder/webapp/api/utils.py#L18
 
+
 def create_parser():
     parser = argparse.ArgumentParser()
     parser.add_argument("branch", action="store",
@@ -31,6 +29,7 @@ def create_parser():
 
     return parser
 
+
 def download(url, prefix, dest, force_suffix=True):
     if dest is None:
         dest = "."
@@ -51,12 +50,25 @@ def download(url, prefix, dest, force_suffix=True):
         for chunk in resp.iter_content(1024):
             f.write(chunk)
 
+
+def fetch_json(url, params=None):
+    headers = {
+        'Accept': 'application/json',
+        'User-Agent': 'wpt-fetchlogs',
+    }
+    response = requests.get(url=url, params=params, headers=headers, timeout=30)
+    response.raise_for_status()
+    return response.json()
+
+
 def get_blobber_url(branch, job):
     job_id = job["id"]
-    resp = requests.get(urlparse.urljoin(treeherder_base,
-                                         "/api/project/%s/artifact/?job_id=%i&name=Job%%20Info" % (branch,
-                                                                                                   job_id)))
-    job_data = resp.json()
+    artifact_url = urlparse.urljoin(treeherder_base, "/api/project/%s/artifact/" % branch)
+    artifact_params = {
+        'job_id': job_id,
+        'name': 'Job Info',
+    }
+    job_data = fetch_json(artifact_url, params=artifact_params)
 
     if job_data:
         assert len(job_data) == 1
@@ -71,15 +83,20 @@ def get_blobber_url(branch, job):
 
 
 def get_structured_logs(branch, commit, dest=None):
-    resp = requests.get(urlparse.urljoin(treeherder_base, "/api/project/%s/resultset/?revision=%s" % (branch, commit)))
-
-    revision_data = resp.json()
-
+    resultset_url = urlparse.urljoin(treeherder_base, "/api/project/%s/resultset/" % branch)
+    resultset_params = {
+        'revision': commit,
+    }
+    revision_data = fetch_json(resultset_url, params=resultset_params)
     result_set = revision_data["results"][0]["id"]
 
-    resp = requests.get(urlparse.urljoin(treeherder_base, "/api/project/%s/jobs/?result_set_id=%s&count=2000&exclusion_profile=false" % (branch, result_set)))
-
-    job_data = resp.json()
+    jobs_url = urlparse.urljoin(treeherder_base, "/api/project/%s/jobs/" % branch)
+    jobs_params = {
+        'result_set_id': result_set,
+        'count': 2000,
+        'exclusion_profile': 'false',
+    }
+    job_data = fetch_json(jobs_url, params=jobs_params)
 
     for result in job_data["results"]:
         job_type_name = result["job_type_name"]
@@ -89,6 +106,7 @@ def get_structured_logs(branch, commit, dest=None):
                 prefix = result["platform"] # platform
                 download(url, prefix, None)
 
+
 def main():
     parser = create_parser()
     args = parser.parse_args()