Fix Perfherder cycling tests

tests/model/cycle_data/test_perfherder_cycling.py

@@ -4,6 +4,7 @@ from unittest.mock import MagicMock
 from unittest.mock import patch
 
 import pytest
+from django.conf import settings
 from django.core.management import call_command
 from django.db import connection, IntegrityError
 
@@ -809,13 +810,16 @@ def test_deleting_performance_data_cascades_to_perf_multicomit_data(test_perf_da
 
     try:
         cursor = connection.cursor()
-        cursor.execute(
-            '''
-            DELETE FROM `performance_datum`
-            WHERE id = %s
-            ''',
-            [perf_datum.id],
-        )
+        if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.mysql':
+            cursor.execute(
+                '''
+                DELETE FROM `performance_datum`
+                WHERE id = %s
+                ''',
+                [perf_datum.id],
+            )
+        else:
+            PerformanceDatum.objects.filter(id=perf_datum.id).delete()
     except IntegrityError:
         pytest.fail()
     finally:

treeherder/model/data_cycling/removal_strategies.py

@@ -6,6 +6,7 @@ from datetime import timedelta, datetime
 from itertools import cycle
 from typing import List
 
+from django.conf import settings
 from django.db.backends.utils import CursorWrapper
 
 from treeherder.model.models import Repository
@@ -81,20 +82,28 @@ class MainRemovalStrategy(RemovalStrategy):
     def remove(self, using: CursorWrapper):
         chunk_size = self._find_ideal_chunk_size()
 
-        # Django's queryset API doesn't support MySQL's
-        # DELETE statements with LIMIT constructs,
-        # even though this database is capable of doing that.
-        #
-        # If ever this support is added in Django, replace
-        # raw SQL bellow with equivalent queryset commands.
-        using.execute(
-            '''
-            DELETE FROM `performance_datum`
-            WHERE push_timestamp <= %s
-            LIMIT %s
-        ''',
-            [self._max_timestamp, chunk_size],
-        )
+        if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.mysql':
+            # Django's queryset API doesn't support MySQL's
+            # DELETE statements with LIMIT constructs,
+            # even though this database is capable of doing that.
+            #
+            # If ever this support is added in Django, replace
+            # raw SQL bellow with equivalent queryset commands.
+            using.execute(
+                '''
+                DELETE FROM `performance_datum`
+                WHERE push_timestamp <= %s
+                LIMIT %s
+            ''',
+                [self._max_timestamp, chunk_size],
+            )
+        else:
+            deleted, _ = PerformanceDatum.objects.filter(
+                id__in=PerformanceDatum.objects.filter(
+                    push_timestamp__lte=self._max_timestamp
+                ).values_list('id')[:chunk_size]
+            ).delete()
+            using.rowcount = deleted
 
     @property
     def name(self) -> str:
@@ -184,25 +193,34 @@ class TryDataRemoval(RemovalStrategy):
         return 'try data removal strategy'
 
     def __attempt_remove(self, using):
-        # Django's queryset API doesn't support MySQL's
-        # DELETE statements with LIMIT constructs,
-        # even though this database is capable of doing that.
-        #
-        # If ever this support is added in Django, replace
-        # raw SQL bellow with equivalent queryset commands.
-        total_signatures = len(self.target_signatures)
-        from_target_signatures = ' OR '.join(['signature_id =  %s'] * total_signatures)
+        if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.mysql':
+            # Django's queryset API doesn't support MySQL's
+            # DELETE statements with LIMIT constructs,
+            # even though this database is capable of doing that.
+            #
+            # If ever this support is added in Django, replace
+            # raw SQL bellow with equivalent queryset commands.
+            total_signatures = len(self.target_signatures)
+            from_target_signatures = ' OR '.join(['signature_id =  %s'] * total_signatures)
 
-        delete_try_data = f'''
-            DELETE FROM `performance_datum`
-            WHERE repository_id = %s AND push_timestamp <= %s AND ({from_target_signatures})
-            LIMIT %s
-        '''
+            delete_try_data = f'''
+                DELETE FROM `performance_datum`
+                WHERE repository_id = %s AND push_timestamp <= %s AND ({from_target_signatures})
+                LIMIT %s
+            '''
 
-        using.execute(
-            delete_try_data,
-            [self.try_repo, self._max_timestamp, *self.target_signatures, self._chunk_size],
-        )
+            using.execute(
+                delete_try_data,
+                [self.try_repo, self._max_timestamp, *self.target_signatures, self._chunk_size],
+            )
+        else:
+            PerformanceDatum.objects.filter(
+                id__in=PerformanceDatum.objects.filter(
+                    repository_id=self.try_repo,
+                    push_timestamp__lte=self._max_timestamp,
+                    signature_id__in=self.target_signatures,
+                ).values_list('id')[: self._chunk_size]
+            ).delete()
 
     def __lookup_new_signature(self):
         self.__target_signatures = self.__try_signatures[: self.SIGNATURE_BULK_SIZE]
@@ -267,24 +285,31 @@ class IrrelevantDataRemoval(RemovalStrategy):
     def remove(self, using: CursorWrapper):
         chunk_size = self._find_ideal_chunk_size()
 
-        # Django's queryset API doesn't support MySQL's
-        # DELETE statements with LIMIT constructs,
-        # even though this database is capable of doing that.
-        #
-        # If ever this support is added in Django, replace
-        # raw SQL bellow with equivalent queryset commands.
-        using.execute(
-            '''
-                DELETE FROM `performance_datum`
-                WHERE repository_id = %s AND push_timestamp <= %s
-                LIMIT %s
-            ''',
-            [
-                self.irrelevant_repo,
-                self._max_timestamp,
-                chunk_size,
-            ],
-        )
+        if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.mysql':
+            # Django's queryset API doesn't support MySQL's
+            # DELETE statements with LIMIT constructs,
+            # even though this database is capable of doing that.
+            #
+            # If ever this support is added in Django, replace
+            # raw SQL bellow with equivalent queryset commands.
+            using.execute(
+                '''
+                    DELETE FROM `performance_datum`
+                    WHERE repository_id = %s AND push_timestamp <= %s
+                    LIMIT %s
+                ''',
+                [
+                    self.irrelevant_repo,
+                    self._max_timestamp,
+                    chunk_size,
+                ],
+            )
+        else:
+            PerformanceDatum.objects.filter(
+                id__in=PerformanceDatum.objects.filter(
+                    repository_id=self.irrelevant_repo, push_timestamp__lte=self._max_timestamp
+                ).values_list('id')[:chunk_size]
+            ).delete()
 
     def _find_ideal_chunk_size(self) -> int:
         max_id_of_non_expired_row = (
@@ -376,25 +401,34 @@ class StalledDataRemoval(RemovalStrategy):
         return 'stalled data removal strategy'
 
     def __attempt_remove(self, using: CursorWrapper):
-        # Django's queryset API doesn't support MySQL's
-        # DELETE statements with LIMIT constructs,
-        # even though this database is capable of doing that.
-        #
-        # If ever this support is added in Django, replace
-        # raw SQL bellow with equivalent queryset commands.
-        using.execute(
-            '''
-                DELETE FROM `performance_datum`
-                WHERE repository_id = %s AND signature_id = %s AND push_timestamp <= %s
-                LIMIT %s
-            ''',
-            [
-                self.target_signature.repository_id,
-                self.target_signature.id,
-                self._max_timestamp,
-                self._chunk_size,
-            ],
-        )
+        if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.mysql':
+            # Django's queryset API doesn't support MySQL's
+            # DELETE statements with LIMIT constructs,
+            # even though this database is capable of doing that.
+            #
+            # If ever this support is added in Django, replace
+            # raw SQL bellow with equivalent queryset commands.
+            using.execute(
+                '''
+                    DELETE FROM `performance_datum`
+                    WHERE repository_id = %s AND signature_id = %s AND push_timestamp <= %s
+                    LIMIT %s
+                ''',
+                [
+                    self.target_signature.repository_id,
+                    self.target_signature.id,
+                    self._max_timestamp,
+                    self._chunk_size,
+                ],
+            )
+        else:
+            PerformanceDatum.objects.filter(
+                id__in=PerformanceDatum.objects.filter(
+                    repository_id=self.target_signature.repository_id,
+                    signature_id=self.target_signature.id,
+                    push_timestamp__lte=self._max_timestamp,
+                ).values_list('id')[: self._chunk_size]
+            ).delete()
 
     def __lookup_new_signature(self):
         try: