diff --git a/treeherder/etl/common.py b/treeherder/etl/common.py index 1eb7c4a41..56ed42e41 100644 --- a/treeherder/etl/common.py +++ b/treeherder/etl/common.py @@ -55,7 +55,10 @@ class JobData(dict): def retrieve_api_content(url): req = urllib2.Request(url) req.add_header('Content-Type', 'application/json') - conn = urllib2.urlopen(req) + conn = urllib2.urlopen( + req, + timeout=settings.TREEHERDER_REQUESTS_TIMEOUT + ) if conn.getcode() == 404: return None @@ -65,7 +68,9 @@ def get_remote_content(url): req = urllib2.Request(url) req.add_header('Accept', 'application/json') req.add_header('Content-Type', 'application/json') - conn = urllib2.urlopen(req) + conn = urllib2.urlopen( + req, + timeout=settings.TREEHERDER_REQUESTS_TIMEOUT) if not conn.getcode() == 200: return None diff --git a/treeherder/log_parser/artifactbuildercollection.pyx b/treeherder/log_parser/artifactbuildercollection.pyx index f43f54b71..099682973 100644 --- a/treeherder/log_parser/artifactbuildercollection.pyx +++ b/treeherder/log_parser/artifactbuildercollection.pyx @@ -1,8 +1,9 @@ import urllib2 import gzip import io -import logging from contextlib import closing +from django.conf import settings + from .artifactbuilders import (BuildbotLogViewArtifactBuilder, BuildbotJobArtifactBuilder, BuildbotPerformanceDataArtifactBuilder) @@ -85,7 +86,10 @@ BuildbotPerformanceDataArtifactBuilder def get_log_handle(self, url): """Hook to get a handle to the log with this url""" - return urllib2.urlopen(url) + return urllib2.urlopen( + url, + timeout=settings.TREEHERDER_REQUESTS_TIMEOUT + ) def parse(self): """ diff --git a/treeherder/model/derived/refdata.py b/treeherder/model/derived/refdata.py index 6b775904f..60cf0ae39 100644 --- a/treeherder/model/derived/refdata.py +++ b/treeherder/model/derived/refdata.py @@ -1256,7 +1256,9 @@ class RefDataManager(object): milestone_path = '/raw-file/default/config/milestone.txt' version_url = "".join((repo_url, milestone_path)) - response = urllib2.urlopen(version_url) + response = urllib2.urlopen( + version_url, + timeout=settings.TREEHERDER_REQUESTS_TIMEOUT) for line in response: #go to the last line pass diff --git a/treeherder/settings/base.py b/treeherder/settings/base.py index 16cfea8e5..b694b29a7 100644 --- a/treeherder/settings/base.py +++ b/treeherder/settings/base.py @@ -235,6 +235,10 @@ SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') TBPL_BUGS_TRANSFER_ENABLED = True TBPL_HOST = "https://tbpl.mozilla.org" +# timeout for requests to external sources +# like ftp.mozilla.org or hg.mozilla.org +TREEHERDER_REQUESTS_TIMEOUT = 30 + try: from .local import * except ImportError: diff --git a/treeherder/webapp/api/logslice.py b/treeherder/webapp/api/logslice.py index 84c9ad079..8ea2ee88f 100644 --- a/treeherder/webapp/api/logslice.py +++ b/treeherder/webapp/api/logslice.py @@ -3,8 +3,8 @@ from rest_framework.response import Response from django.core import cache from treeherder.webapp.api.utils import (with_jobs) - from treeherder.webapp.api.exceptions import ResourceNotFoundException +from django.conf import settings import urllib2 import gzip @@ -20,7 +20,10 @@ class LogSliceView(viewsets.ViewSet): def get_log_handle(self, url): """Hook to get a handle to the log with this url""" - return urllib2.urlopen(url) + return urllib2.urlopen( + url, + timeout=settings.TREEHERDER_REQUESTS_TIMEOUT + ) @with_jobs def list(self, request, project, jm):