Merge pull request #185 from imjalpreet/Bug-1128601

Fix Bug 1128601 - Retry on http get errors in mozregression

mozregression/build_data.py

@@ -7,7 +7,7 @@ import threading
 import datetime
 
 from mozregression import errors
-from mozregression.utils import url_links, get_http_session
+from mozregression.utils import url_links, retry_get
 
 
 class BuildData(object):
@@ -257,7 +257,7 @@ class BuildFolderInfoFetcher(object):
         is found.
         """
         data = {}
-        response = get_http_session().get(url)
+        response = retry_get(url)
         for line in response.text.splitlines():
             if '/rev/' in line:
                 repository, changeset = line.split('/rev/')
@@ -336,12 +336,12 @@ class PushLogsFinder(object):
         chset_url = '%s/json-pushes?changeset=%s' % (
             self.get_repo_url(),
             self.start_rev)
-        response = get_http_session().get(chset_url)
+        response = retry_get(chset_url)
         response.raise_for_status()
         chsets = response.json()
 
         # now fetch all remaining changesets
-        response = get_http_session().get(self.pushlog_url())
+        response = retry_get(self.pushlog_url())
         response.raise_for_status()
         chsets.update(response.json())
         # sort pushlogs by date

mozregression/utils.py

@@ -14,6 +14,7 @@ from BeautifulSoup import BeautifulSoup
 import mozinfo
 import requests
 import mozfile
+import redo
 
 from mozregression import errors
 
@@ -60,6 +61,13 @@ class ClassRegistry(object):
 CACHE_SESSION = None
 
 
+def retry_get(url, **kwargs):
+    return redo.retry(get_http_session().get, attempts=3, sleeptime=1,
+                      retry_exceptions=(requests.exceptions.HTTPError,
+                                        requests.exceptions.ConnectionError),
+                      args=(url,), kwargs=kwargs)
+
+
 def set_http_cache_session(cache_session, get_defaults=None):
     """
     Define a cache http session.
@@ -162,7 +170,7 @@ def url_links(url, regex=None, auth=None):
     """
     Returns a list of links that can be found on a given web page.
     """
-    response = get_http_session().get(url, auth=auth)
+    response = retry_get(url, auth=auth)
     response.raise_for_status()
 
     soup = BeautifulSoup(response.text)

setup.py

@@ -34,6 +34,7 @@ setup(name="mozregression",
           'mozlog >= 2.7',
           'mozversion >= 1.2',
           'requests >= 2.5.0',
+          'redo',
           'cachecontrol >= 0.10.2',
           # used in conjunction with cachecontrol
           'lockfile >= 0.10.2',