Revert "Replace custom retry function with tenacity retrying library (#1235)"

This reverts commit 9e8d27ec1f.

.isort.cfg

@@ -4,4 +4,4 @@ include_trailing_comma=True
 force_grid_wrap=0
 use_parentheses=True
 line_length=88
-known_third_party = apispec,apispec_webframeworks,cerberus,dateutil,flask,flask_cors,hglib,imblearn,joblib,jsone,jsonschema,libmozdata,lmdb,marshmallow,matplotlib,microannotate,models,numpy,orjson,pandas,pkg_resources,pydriller,pyemd,pytest,redis,requests,responses,rq,scipy,setuptools,shap,sklearn,tabulate,taskcluster,tenacity,tqdm,xgboost,yaml,zstandard
+known_third_party = apispec,apispec_webframeworks,cerberus,dateutil,flask,flask_cors,hglib,imblearn,joblib,jsone,jsonschema,libmozdata,lmdb,marshmallow,matplotlib,microannotate,models,numpy,orjson,pandas,pkg_resources,pydriller,pyemd,pytest,redis,requests,responses,rq,scipy,setuptools,shap,sklearn,tabulate,taskcluster,tqdm,xgboost,yaml,zstandard

bugbug/utils.py

@@ -9,6 +9,7 @@ import logging
 import os
 import socket
 import tarfile
+import time
 from collections import deque
 from contextlib import contextmanager
 
@@ -203,6 +204,18 @@ def download_and_load_model(model_name):
     return get_model_class(model_name).load(path)
 
 
+def retry(operation, retries=5, wait_between_retries=30):
+    while True:
+        try:
+            return operation()
+        except Exception:
+            retries -= 1
+            if retries == 0:
+                raise
+
+            time.sleep(wait_between_retries)
+
+
 def zstd_compress(path):
     cctx = zstandard.ZstdCompressor()
     with open(path, "rb") as input_f:

requirements.txt

@@ -18,7 +18,6 @@ scipy==1.4.1
 shap[plots]==0.33.0
 tabulate==0.8.6
 taskcluster==24.1.10
-tenacity==6.0.0
 tqdm==4.41.1
 xgboost==0.90
 zstandard==0.13.0

scripts/commit_classifier.py

@@ -18,7 +18,6 @@ import joblib
 import matplotlib
 import numpy as np
 import shap
-import tenacity
 from dateutil.relativedelta import relativedelta
 from libmozdata import vcs_map
 from libmozdata.phabricator import PhabricatorAPI
@@ -29,6 +28,7 @@ from bugbug.utils import (
     download_and_load_model,
     download_check_etag,
     get_secret,
+    retry,
     to_array,
     zstd_decompress,
 )
@@ -191,31 +191,25 @@ class CommitClassifier(object):
         logger.info(f"Cloning {repo_url}...")
 
         if not os.path.exists(repo_dir):
-            tenacity.retry(
+            retry(
                 lambda: subprocess.run(
                     ["git", "clone", "--quiet", repo_url, repo_dir], check=True
-                ),
-                wait=tenacity.wait_fixed(30),
-                stop=tenacity.stop_after_attempt(5),
+                )
             )
 
-        tenacity.retry(
+        retry(
             lambda: subprocess.run(
                 ["git", "pull", "--quiet", repo_url, "master"],
                 cwd=repo_dir,
                 capture_output=True,
                 check=True,
-            ),
-            wait=tenacity.wait_fixed(30),
-            stop=tenacity.stop_after_attempt(5),
+            )
         )
 
-        tenacity.retry(
+        retry(
             lambda: subprocess.run(
                 ["git", "checkout", rev], cwd=repo_dir, capture_output=True, check=True
-            ),
-            wait=tenacity.wait_fixed(30),
-            stop=tenacity.stop_after_attempt(5),
+            )
         )
 
     def update_commit_db(self):

scripts/microannotate_generator.py

@@ -5,11 +5,10 @@ import os
 import subprocess
 from logging import INFO, basicConfig, getLogger
 
-import tenacity
 from microannotate import generator
 
 from bugbug import db, repository
-from bugbug.utils import ThreadPoolExecutorResult, get_secret
+from bugbug.utils import ThreadPoolExecutorResult, get_secret, retry
 
 basicConfig(level=INFO)
 logger = getLogger(__name__)
@@ -62,12 +61,10 @@ class MicroannotateGenerator(object):
             else:
                 executor.submit(self.init_git_repo)
 
-        tenacity.retry(
+        retry(
             lambda: subprocess.run(
                 ["git", "config", "--global", "http.postBuffer", "12M"], check=True
-            ),
-            wait=tenacity.wait_fixed(30),
-            stop=tenacity.stop_after_attempt(5),
+            )
         )
 
         push_args = ["git", "push", repo_push_url, "master"]
@@ -84,11 +81,7 @@ class MicroannotateGenerator(object):
                 remove_comments=self.remove_comments,
             )
 
-            tenacity.retry(
-                lambda: subprocess.run(push_args, cwd=self.git_repo_path, check=True),
-                wait=tenacity.wait_fixed(30),
-                stop=tenacity.stop_after_attempt(5),
-            )
+            retry(lambda: subprocess.run(push_args, cwd=self.git_repo_path, check=True))
 
     def init_git_repo(self):
         subprocess.run(["git", "init", self.git_repo_path], check=True)
@@ -100,25 +93,21 @@ class MicroannotateGenerator(object):
         )
 
     def clone_git_repo(self):
-        tenacity.retry(
+        retry(
             lambda: subprocess.run(
                 ["git", "clone", "--quiet", self.repo_url, self.git_repo_path],
                 check=True,
-            ),
-            wait=tenacity.wait_fixed(30),
-            stop=tenacity.stop_after_attempt(5),
+            )
         )
 
         try:
-            tenacity.retry(
+            retry(
                 lambda: subprocess.run(
                     ["git", "pull", "--quiet", self.repo_url, "master"],
                     cwd=self.git_repo_path,
                     capture_output=True,
                     check=True,
-                ),
-                wait=tenacity.wait_fixed(30),
-                stop=tenacity.stop_after_attempt(5),
+                )
             )
         except subprocess.CalledProcessError as e:
             # When the repo is empty.

scripts/regressor_finder.py

@@ -14,7 +14,6 @@ from logging import INFO, basicConfig, getLogger
 
 import dateutil.parser
 import hglib
-import tenacity
 from dateutil.relativedelta import relativedelta
 from libmozdata import vcs_map
 from microannotate import utils as microannotate_utils
@@ -27,7 +26,7 @@ from bugbug.models.regressor import (
     BUG_INTRODUCING_COMMITS_DB,
     TOKENIZED_BUG_INTRODUCING_COMMITS_DB,
 )
-from bugbug.utils import download_and_load_model, zstd_compress
+from bugbug.utils import download_and_load_model, retry, zstd_compress
 
 basicConfig(level=INFO)
 logger = getLogger(__name__)
@@ -84,23 +83,19 @@ class RegressorFinder(object):
 
     def clone_git_repo(self, repo_url, repo_dir):
         if not os.path.exists(repo_dir):
-            tenacity.retry(
+            retry(
                 lambda: subprocess.run(
                     ["git", "clone", "--quiet", repo_url, repo_dir], check=True
-                ),
-                wait=tenacity.wait_fixed(30),
-                stop=tenacity.stop_after_attempt(5),
+                )
             )
 
-        tenacity.retry(
+        retry(
             lambda: subprocess.run(
                 ["git", "pull", "--quiet", repo_url, "master"],
                 cwd=repo_dir,
                 capture_output=True,
                 check=True,
-            ),
-            wait=tenacity.wait_fixed(30),
-            stop=tenacity.stop_after_attempt(5),
+            )
         )
 
     def init_mapping(self):