Merge pull request #168 from mozilla/set-urlopen-timeout

set timeout for every request to external services

treeherder/etl/common.py

@@ -55,7 +55,10 @@ class JobData(dict):
 def retrieve_api_content(url):
     req = urllib2.Request(url)
     req.add_header('Content-Type', 'application/json')
-    conn = urllib2.urlopen(req)
+    conn = urllib2.urlopen(
+        req,
+        timeout=settings.TREEHERDER_REQUESTS_TIMEOUT
+    )
     if conn.getcode() == 404:
         return None
 
@@ -65,7 +68,9 @@ def get_remote_content(url):
     req = urllib2.Request(url)
     req.add_header('Accept', 'application/json')
     req.add_header('Content-Type', 'application/json')
-    conn = urllib2.urlopen(req)
+    conn = urllib2.urlopen(
+        req,
+        timeout=settings.TREEHERDER_REQUESTS_TIMEOUT)
 
     if not conn.getcode() == 200:
         return None

treeherder/log_parser/artifactbuildercollection.pyx

@@ -1,8 +1,9 @@
 import urllib2
 import gzip
 import io
-import logging
 from contextlib import closing
+from django.conf import settings
+
 from .artifactbuilders import (BuildbotLogViewArtifactBuilder,
                                BuildbotJobArtifactBuilder,
                                BuildbotPerformanceDataArtifactBuilder)
@@ -85,7 +86,10 @@ BuildbotPerformanceDataArtifactBuilder
 
     def get_log_handle(self, url):
         """Hook to get a handle to the log with this url"""
-        return urllib2.urlopen(url)
+        return urllib2.urlopen(
+               url,
+               timeout=settings.TREEHERDER_REQUESTS_TIMEOUT
+        )
 
     def parse(self):
         """

treeherder/model/derived/refdata.py

@@ -1256,7 +1256,9 @@ class RefDataManager(object):
         milestone_path = '/raw-file/default/config/milestone.txt'
         version_url = "".join((repo_url, milestone_path))
 
-        response = urllib2.urlopen(version_url)
+        response = urllib2.urlopen(
+            version_url,
+            timeout=settings.TREEHERDER_REQUESTS_TIMEOUT)
         for line in response:
             #go to the last line
             pass

treeherder/settings/base.py

@@ -235,6 +235,10 @@ SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
 TBPL_BUGS_TRANSFER_ENABLED = True
 TBPL_HOST = "https://tbpl.mozilla.org"
 
+# timeout for requests to external sources
+# like ftp.mozilla.org or hg.mozilla.org
+TREEHERDER_REQUESTS_TIMEOUT = 30
+
 try:
     from .local import *
 except ImportError:

treeherder/webapp/api/logslice.py

@@ -3,8 +3,8 @@ from rest_framework.response import Response
 from django.core import cache
 
 from treeherder.webapp.api.utils import (with_jobs)
-
 from treeherder.webapp.api.exceptions import ResourceNotFoundException
+from django.conf import settings
 
 import urllib2
 import gzip
@@ -20,7 +20,10 @@ class LogSliceView(viewsets.ViewSet):
 
     def get_log_handle(self, url):
         """Hook to get a handle to the log with this url"""
-        return urllib2.urlopen(url)
+        return urllib2.urlopen(
+            url,
+            timeout=settings.TREEHERDER_REQUESTS_TIMEOUT
+        )
 
     @with_jobs
     def list(self, request, project, jm):