Bug 1663604 - Extend the Bugzilla formulas columns to include the alerts amount for each excel row

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://gitlab.com/pycqa/flake8
-    rev: 3.7.9
+    rev: 3.8.3
     hooks:
       - id: flake8
   - repo: https://github.com/shellcheck-py/shellcheck-py
@@ -8,16 +8,16 @@ repos:
     hooks:
       - id: shellcheck
   - repo: https://github.com/prettier/prettier
-    rev: 2.0.5
+    rev: 2.1.1
     hooks:
       - id: prettier
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.22.0
+    rev: v0.23.2
     hooks:
       - id: markdownlint
         args: [--fix]
   - repo: https://github.com/psf/black
-    rev: stable
+    rev: 20.8b1
     hooks:
       - id: black
         language_version: python3.7

tests/perf/test_sheriffing_criteria/test_common_behaviour.py

@@ -2,7 +2,7 @@ from datetime import datetime, timedelta
 
 import pytest
 from django.conf import settings
-from typing import List, Type
+from typing import List, Type, Callable
 
 from tests.perf.test_sheriffing_criteria.conftest import CASSETTES_RECORDING_DATE
 from treeherder.config.settings import BZ_DATETIME_FORMAT
@@ -11,16 +11,21 @@ from treeherder.perf.sheriffing_criteria import (
     EngineerTractionFormula,
     FixRatioFormula,
     BugzillaFormula,
+    TotalAlertsFormula,
 )
 
 
 pytestmark = [pytest.mark.freeze_time(CASSETTES_RECORDING_DATE, tick=True)]
 
 
-def formula_instances() -> List[BugzillaFormula]:
+def bugzilla_formula_instances() -> List[BugzillaFormula]:
     return [EngineerTractionFormula(), FixRatioFormula()]
 
 
+def formula_instances() -> List[Callable]:
+    return bugzilla_formula_instances() + [TotalAlertsFormula()]
+
+
 def concrete_formula_classes() -> List[Type[BugzillaFormula]]:
     return [EngineerTractionFormula, FixRatioFormula]
 
@@ -30,14 +35,20 @@ def test_formula_exposes_quantifying_period(formula, nonblock_session):
     assert formula.quantifying_period == settings.QUANTIFYING_PERIOD
 
 
-@pytest.mark.parametrize('formula', formula_instances())
+@pytest.mark.parametrize('formula', bugzilla_formula_instances())
 def test_formula_exposes_oldest_timestamp(formula, nonblock_session):
     no_older_than = datetime.now() - timedelta(weeks=24, seconds=5)
 
     assert formula.oldest_timestamp >= no_older_than
 
 
-@pytest.mark.parametrize('formula', formula_instances())
+def test_total_alerts_formula_exposes_oldest_timestamp():
+    no_older_than = datetime.now() - (timedelta(weeks=24, seconds=5) + timedelta(weeks=2))
+
+    assert TotalAlertsFormula().oldest_timestamp >= no_older_than
+
+
+@pytest.mark.parametrize('formula', bugzilla_formula_instances())
 @pytest.mark.parametrize(
     'cooled_down_bug',
     [
@@ -50,7 +61,7 @@ def test_formula_correctly_detects_cooled_down_bugs(cooled_down_bug, formula, no
     assert formula.has_cooled_down(cooled_down_bug)
 
 
-@pytest.mark.parametrize('formula', formula_instances())
+@pytest.mark.parametrize('formula', bugzilla_formula_instances())
 @pytest.mark.parametrize(
     'not_cooled_down_bug',
     [
@@ -65,7 +76,7 @@ def test_formula_detects_bugs_that_didnt_cool_down_yet(
     assert not formula.has_cooled_down(not_cooled_down_bug)
 
 
-@pytest.mark.parametrize('formula', formula_instances())
+@pytest.mark.parametrize('formula', bugzilla_formula_instances())
 @pytest.mark.parametrize('bad_structured_bug', [{}, {'creation_time': 'jiberish-date'}])
 def test_formula_throws_adequate_error_for_bug(bad_structured_bug, formula, nonblock_session):
     with pytest.raises(ValueError):
@@ -91,7 +102,7 @@ def test_formula_cannot_be_initialized_with_a_regular_session(FormulaClass, unre
         _ = FormulaClass(unrecommended_session)
 
 
-@pytest.mark.parametrize('formula', formula_instances())
+@pytest.mark.parametrize('formula', bugzilla_formula_instances())
 def test_accessing_breakdown_without_prior_calculus_errors_out(formula, nonblock_session):
     with pytest.raises(RuntimeError):
         _ = formula.breakdown()

tests/perf/test_sheriffing_criteria/test_criteria_tracker.py

@@ -19,6 +19,7 @@ from treeherder.perf.sheriffing_criteria import (
     FixRatioFormula,
     RecordComputer,
     CriteriaRecord,
+    TotalAlertsFormula,
 )
 from treeherder.perf.sheriffing_criteria.criteria_tracking import ResultsChecker
 from treeherder.utils import PROJECT_ROOT
@@ -56,6 +57,7 @@ RECORDS_WITH_NO_DATA = [
         Test=test[2],
         EngineerTraction='',
         FixRatio='',
+        TotalAlerts='',
         LastUpdatedOn='',
         AllowSync='',
     )
@@ -68,6 +70,7 @@ RECORDS_WITH_EXPIRED_DATA = [
         Test=test[2],
         EngineerTraction=0.5,
         FixRatio=0.3,
+        TotalAlerts=21,
         LastUpdatedOn='2020-05-02T00:00:00.000000',
         AllowSync='',
     )
@@ -80,6 +83,7 @@ RECORDS_WITH_UPDATED_DATA = [
         Test=test[2],
         EngineerTraction=0.5,
         FixRatio=0.3,
+        TotalAlerts=21,
         LastUpdatedOn='2020-06-02T00:00:00.000000',
         AllowSync='',
     )
@@ -158,6 +162,7 @@ def mock_formula_map():
     return {
         'EngineerTraction': MagicMock(spec=EngineerTractionFormula, return_value=EXPECTED_VALUE),
         'FixRatio': MagicMock(spec=FixRatioFormula, return_value=EXPECTED_VALUE),
+        'TotalAlerts': MagicMock(spec=FixRatioFormula, return_value=0),
     }
 
 
@@ -166,7 +171,6 @@ def mock_formula_map():
     [
         {'EngineerTraction': InvalidFormula(), 'FixRatio': InvalidFormula()},
         {'EngineerTraction': None, 'FixRatio': None},
-        {'EngineerTraction': lambda f, s: None, 'FixRatio': lambda f, s: None},
     ],
 )
 def test_tracker_throws_error_for_invalid_formulas(invalid_formulas):
@@ -220,10 +224,11 @@ def test_record_computer_can_tell_unallowed_data(criteria_record):
 
 @pytest.mark.freeze_time(CASSETTES_RECORDING_DATE)  # disable tick
 @pytest.mark.parametrize('exception', [NoFiledBugs(), Exception()])
-def test_record_computer_still_updates_if_one_of_the_formulas_fails(exception):
+def test_record_computer_still_updates_if_one_of_the_formulas_fails(exception, db):
     formula_map = {
         'EngineerTraction': MagicMock(spec=EngineerTractionFormula, return_value=EXPECTED_VALUE),
         'FixRatio': MagicMock(spec=FixRatioFormula, side_effect=exception),
+        'TotalAlerts': TotalAlertsFormula(),
     }
     record = CriteriaRecord(
         Framework='talos',
@@ -231,6 +236,7 @@ def test_record_computer_still_updates_if_one_of_the_formulas_fails(exception):
         Test='',
         EngineerTraction='',
         FixRatio='',
+        TotalAlerts='',
         LastUpdatedOn='',
         AllowSync='',
     )
@@ -242,6 +248,7 @@ def test_record_computer_still_updates_if_one_of_the_formulas_fails(exception):
     assert record.Suite == 'tp5n'
     assert record.EngineerTraction == EXPECTED_VALUE
     assert record.FixRatio == 'N/A'
+    assert record.TotalAlerts == 0  # as the test database is empty
     assert record.LastUpdatedOn == EXPECTED_LAST_UPDATE
     assert record.AllowSync is True
 
@@ -272,6 +279,7 @@ def test_tracker_updates_records_with_missing_data(mock_formula_map, updatable_c
     for criteria_rec in tracker:
         assert criteria_rec.EngineerTraction == ''
         assert criteria_rec.FixRatio == ''
+        assert criteria_rec.TotalAlerts == ''
         assert criteria_rec.LastUpdatedOn == ''
         assert criteria_rec.AllowSync is True
 
@@ -286,6 +294,7 @@ def test_tracker_updates_records_with_missing_data(mock_formula_map, updatable_c
     for criteria_rec in separate_tracker:
         assert criteria_rec.EngineerTraction == EXPECTED_VALUE
         assert criteria_rec.FixRatio == EXPECTED_VALUE
+        assert criteria_rec.TotalAlerts == 0
         assert criteria_rec.LastUpdatedOn == EXPECTED_LAST_UPDATE
         assert criteria_rec.AllowSync is True
 

tests/sample_data/criteria-records.csv

@@ -1,6 +1,6 @@
-Framework,Suite,Test,EngineerTraction,FixRatio,LastUpdatedOn,AllowSync
-awsy,JS,,,,,
-build_metrics,build times,,,,,
-build_metrics,installer size,,,,,
-raptor,raptor-speedometer-firefox,,,,,
-raptor,raptor-webaudio-firefox,,,,,
+Framework,Suite,Test,EngineerTraction,FixRatio,TotalAlerts,LastUpdatedOn,AllowSync
+awsy,JS,,,,,,
+build_metrics,build times,,,,,,
+build_metrics,installer size,,,,,,
+raptor,raptor-speedometer-firefox,,,,,,
+raptor,raptor-webaudio-firefox,,,,,,

treeherder/perf/management/commands/compute_criteria_formulas.py

@@ -8,6 +8,7 @@ from treeherder.perf.sheriffing_criteria import (
     EngineerTractionFormula,
     FixRatioFormula,
     CriteriaTracker,
+    TotalAlertsFormula,
 )
 from treeherder.perf.sheriffing_criteria import criteria_tracking
 from mo_times import Duration
@@ -84,6 +85,7 @@ class Command(BaseCommand):
         formula_map = {
             'EngineerTraction': EngineerTractionFormula(*init_params),
             'FixRatio': FixRatioFormula(*init_params),
+            'TotalAlerts': TotalAlertsFormula(quant_period),
         }
 
         tracker = CriteriaTracker(formula_map, multiprocessed=multiprocessed)

treeherder/perf/sheriffing_criteria/__init__.py

@@ -3,5 +3,6 @@ from .bugzilla_formulas import (  # noqa
     BugzillaFormula,
     EngineerTractionFormula,
     FixRatioFormula,
+    TotalAlertsFormula,
 )
 from .criteria_tracking import CriteriaTracker, RecordComputer, CriteriaRecord  # noqa

treeherder/perf/sheriffing_criteria/bugzilla_formulas.py

@@ -9,6 +9,7 @@ from requests import Session
 
 from treeherder.config.settings import BZ_DATETIME_FORMAT
 from treeherder.perf.exceptions import NoFiledBugs, BugzillaEndpointError
+from treeherder.perf.models import PerformanceAlert
 
 # Google Doc specification
 PERF_SHERIFFING_CRITERIA = (
@@ -226,3 +227,36 @@ class FixRatioFormula(BugzillaFormula):
 
     def _create_default_session(self) -> NonBlockableSession:
         return NonBlockableSession(referer=f'{FIX_RATIO_SPECIFICATION}')
+
+
+class TotalAlertsFormula:
+    MAX_INVESTIGATION_TIME = timedelta(
+        weeks=2
+    )  # until perf sheriffs should figure out a particular culprit
+
+    def __init__(
+        self,
+        quantifying_period: timedelta = None,
+    ):
+        self._quant_period = quantifying_period or settings.QUANTIFYING_PERIOD
+
+    @property
+    def quantifying_period(self):
+        return self._quant_period
+
+    @property
+    def oldest_timestamp(self):
+        return datetime.now() - (self._quant_period + self.MAX_INVESTIGATION_TIME)
+
+    def __call__(self, framework: str, suite: str, test: str = None) -> int:
+        filters = {'series_signature__framework__name': framework, 'series_signature__suite': suite}
+        if test is not None:
+            filters['series_signature__test'] = test
+
+        return (
+            PerformanceAlert.objects.select_related(
+                'series_signature', 'series_signature__framework'
+            )
+            .filter(**filters, last_updated__gte=self.oldest_timestamp)
+            .count()
+        )

treeherder/perf/sheriffing_criteria/criteria_tracking.py

@@ -24,6 +24,7 @@ class CriteriaRecord:
     Test: str
     EngineerTraction: Union[float, str]
     FixRatio: Union[float, str]
+    TotalAlerts: int
     LastUpdatedOn: datetime
     AllowSync: bool
 
@@ -32,6 +33,8 @@ class CriteriaRecord:
             self.EngineerTraction = float(self.EngineerTraction)
         if self.FixRatio not in ('', 'N/A'):
             self.FixRatio = float(self.FixRatio)
+        if self.TotalAlerts not in ('', 'N/A'):
+            self.TotalAlerts = int(self.TotalAlerts)
 
         if self.LastUpdatedOn != '':
             if isinstance(self.LastUpdatedOn, str):
@@ -230,10 +233,8 @@ class CriteriaTracker:
         self._records_map = {}
 
         for formula in self._formula_map.values():
-            if not isinstance(formula, BugzillaFormula):
-                raise TypeError(
-                    f'Must provide formulas of type {BugzillaFormula.__class__.__name__}'
-                )
+            if not callable(formula):
+                raise TypeError('Must provide callable as sheriffing criteria formula')
 
     def get_test_moniker(self, record: CriteriaRecord) -> Tuple[str, str, str]:
         return record.Framework, record.Suite, record.Test