Bug 1618840 - Create secretary tool for PerfSheriffBot

tests/perfalert/test_alerts/test_secretary_tool.py

@@ -0,0 +1,91 @@
+from datetime import (datetime,
+                      timedelta)
+
+import pytest
+import simplejson as json
+from mock import (Mock,
+                  patch)
+
+from treeherder.perf.models import (BackfillRecord,
+                                    BackfillReport,
+                                    PerformanceSettings)
+from treeherder.perf.secretary_tool import (SecretaryTool,
+                                            default_serializer)
+
+
+@pytest.fixture
+def performance_settings(db):
+    settings = {
+            "limits": 500,
+            "last_reset_date": datetime.utcnow(),
+        }
+    return PerformanceSettings.objects.create(
+        name="perf_sheriff_bot",
+        settings=json.dumps(settings, default=default_serializer),
+    )
+
+
+@pytest.fixture
+def expired_performance_settings(db):
+    settings = {
+            "limits": 500,
+            "last_reset_date": datetime.utcnow() - timedelta(days=30),
+        }
+    return PerformanceSettings.objects.create(
+        name="perf_sheriff_bot",
+        settings=json.dumps(settings, default=default_serializer),
+    )
+
+
+@pytest.fixture
+def create_record():
+    def _create_record(alert):
+        report = BackfillReport.objects.create(summary=alert.summary)
+        return BackfillRecord.objects.create(alert=alert, report=report)
+
+    return _create_record
+
+
+def test_secretary_tool_updates_only_matured_reports(test_perf_alert, test_perf_alert_2, create_record):
+    # create new report with records
+    create_record(test_perf_alert)
+    # create mature report with records
+    date_past = datetime.utcnow() - timedelta(hours=5)
+    with patch('django.utils.timezone.now', Mock(return_value=date_past)):
+        create_record(test_perf_alert_2)
+
+    assert BackfillRecord.objects.count() == 2
+    assert BackfillRecord.objects.filter(status=BackfillRecord.PRELIMINARY).count() == 2
+
+    SecretaryTool.mark_reports_for_backfill()
+    assert BackfillRecord.objects.filter(status=BackfillRecord.PRELIMINARY).count() == 1
+
+
+def test_secretary_tool_uses_existing_settings(performance_settings):
+    assert PerformanceSettings.objects.count() == 1
+    last_reset_date_before = json.loads(performance_settings.settings)["last_reset_date"]
+
+    SecretaryTool.validate_settings()
+
+    assert PerformanceSettings.objects.count() == 1
+    settings_after = PerformanceSettings.objects.filter(name="perf_sheriff_bot").first()
+    assert json.loads(settings_after.settings)["last_reset_date"] == last_reset_date_before
+
+
+def test_secretary_tool_resets_settings_if_expired(expired_performance_settings):
+    assert PerformanceSettings.objects.count() == 1
+    expired_last_reset_date = json.loads(expired_performance_settings.settings)["last_reset_date"]
+
+    SecretaryTool.validate_settings()
+
+    assert PerformanceSettings.objects.count() == 1
+    settings_after = PerformanceSettings.objects.filter(name="perf_sheriff_bot").first()
+    assert json.loads(settings_after.settings)["last_reset_date"] != expired_last_reset_date
+
+
+def test_secretary_tool_creates_new_settings_if_none_exist(db):
+    assert PerformanceSettings.objects.count() == 0
+
+    SecretaryTool.validate_settings()
+
+    assert PerformanceSettings.objects.count() == 1

treeherder/perf/alerts.py

@@ -371,7 +371,8 @@ class BackfillReportMaintainer:
                 continue
 
             backfill_report, created = BackfillReport.objects.get_or_create(summary_id=summary.id)
-            if created or backfill_report.is_outdated:
+            # only provide new records if the report is not frozen
+            if not backfill_report.frozen and (created or backfill_report.is_outdated):
                 backfill_report.expel_records()  # associated records are outdated & irrelevant
                 self._provide_records(backfill_report, alert_context_map)
             reports.append(backfill_report)

treeherder/perf/migrations/0029_add_frozen_to_report.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.0.3 on 2020-03-30 06:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('perf', '0028_default_application_to_empty_str'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='backfillreport',
+            name='frozen',
+            field=models.BooleanField(default=False),
+        ),
+    ]

treeherder/perf/models.py

@@ -528,6 +528,7 @@ class BackfillReport(models.Model):
 
     created = models.DateTimeField(auto_now_add=True)
     last_updated = models.DateTimeField(auto_now=True)
+    frozen = models.BooleanField(default=False)
 
     @property
     def is_outdated(self):

treeherder/perf/secretary_tool.py

@@ -0,0 +1,79 @@
+import logging
+from datetime import (datetime,
+                      timedelta)
+
+import simplejson as json
+from django.conf import settings as django_settings
+
+from treeherder.perf.models import (BackfillRecord,
+                                    BackfillReport,
+                                    PerformanceSettings)
+
+logger = logging.getLogger(__name__)
+
+
+# TODO: update the backfill status using data
+class SecretaryTool:
+    """
+    Tool used for doing the secretary work in the Performance Sheriff Bot.
+    """
+    TIME_TO_MATURE = timedelta(hours=4)
+
+    def __init__(self):
+        pass
+
+    @classmethod
+    def validate_settings(cls):
+        perf_sheriff_settings, created = PerformanceSettings.objects.get_or_create(
+            name="perf_sheriff_bot",
+            defaults={"settings": cls._get_default_settings()},
+        )
+
+        if created:
+            logger.info("Performance settings for perf_sheriff_bot not found. Creating with defaults.")
+            return
+
+        # reset limits if the settings expired
+        settings = json.loads(perf_sheriff_settings.settings)
+        if cls.are_expired(settings):
+            logger.info(f"Settings are expired. Expired settings: {settings}")
+
+            perf_sheriff_settings.settings = cls._get_default_settings()
+            perf_sheriff_settings.save()
+
+    @classmethod
+    def mark_reports_for_backfill(cls):
+        # get the backfill reports that are mature, but not frozen
+        mature_date_limit = datetime.utcnow() - cls.TIME_TO_MATURE
+        mature_reports = BackfillReport.objects.filter(frozen=False, last_updated__lte=mature_date_limit)
+
+        for report in mature_reports:
+            should_freeze = False
+            for record in report.records.all():
+                if record.status == BackfillRecord.PRELIMINARY:
+                    record.status = BackfillRecord.READY_FOR_PROCESSING
+                    record.save()
+                    should_freeze = True
+
+            if should_freeze:
+                report.frozen = True
+                report.save()
+
+    @classmethod
+    def are_expired(cls, settings):
+        last_reset_date = datetime.fromisoformat(settings["last_reset_date"])
+        return datetime.utcnow() > last_reset_date + django_settings.RESET_BACKFILL_LIMITS
+
+    @classmethod
+    def _get_default_settings(cls, as_json=True):
+        default_settings = {
+            "limits": django_settings.MAX_BACKFILLS_PER_PLATFORM,
+            "last_reset_date": datetime.utcnow(),
+        }
+
+        return json.dumps(default_settings, default=default_serializer) if as_json else default_settings
+
+
+def default_serializer(val):
+    if isinstance(val, datetime):
+        return val.isoformat()