Update Django from 4.2 to 5.1 (#8218)

* Bump Django to 5.1.1 and django rest framework to 3.15.2

* Bump Python to 3.10

Also updates old references

* Replace AlterIndexTogether in migrations

https://docs.djangoproject.com/en/4.2/releases/4.2/#index-together-option-is-deprecated-in-favor-of-indexes

* Fix linters deprecation errors

.circleci/config.yml

@@ -28,7 +28,7 @@ jobs:
 
   builds:
     docker:
-      - image: 'cimg/python:3.9-node'
+      - image: 'cimg/python:3.10-node'
     steps:
       - checkout
       - restore_cache:

docker/Dockerfile

@@ -15,7 +15,7 @@ RUN yarn build
 
 
 ## Backend stage
-FROM python:3.9.20-slim-bullseye
+FROM python:3.10.13-slim-bullseye
 
 WORKDIR /app
 

docker/dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9.20-bullseye
+FROM python:3.10.13-bullseye
 
 # Variables that are not specific to a particular environment.
 ENV NEW_RELIC_CONFIG_FILE newrelic.ini

docs/common_tasks.md

@@ -27,8 +27,8 @@ The docs will then be available at: <http://localhost:8000>
 
 ## Debugging Tools
 
-You can use the Python Debugger ([pdb](https://docs.python.org/3.7/library/pdb.html)) in a Docker container.
+You can use the Python Debugger ([pdb](https://docs.python.org/3.10/library/pdb.html)) in a Docker container.
 After starting a local Treeherder instance using [docker-compose](installation.md#server-and-full-stack-development),
 in a separate shell type `docker attach backend`. Then set a breakpoint in your file using either `import pdb; pdb.set_trace()`
-or `breakpoint()` (for Python v3.7+). The pdb debugger will start in that shell once the breakpoint has been triggered.
+or `breakpoint()`. The pdb debugger will start in that shell once the breakpoint has been triggered.
 For example, it can be triggered via refreshing the browser (localhost) if the view you're on calls an API with a breakpoint on it.

manage.py

@@ -5,7 +5,7 @@ import sys
 import warnings
 
 # Display deprecation warnings, which are hidden by default:
-# https://docs.python.org/3.7/library/warnings.html#default-warning-filters
+# https://docs.python.org/3.10/library/warnings.html#default-warning-filter
 warnings.simplefilter("default", DeprecationWarning)
 
 if __name__ == "__main__":

pyproject.toml

@@ -25,8 +25,8 @@ mdx_truly_sane_lists = { version = "1.3", optional = true }
 # Same as Black.
 line-length = 100
 
-# Assume Python 3.9
-target-version = "py39"
+# Assume Python 3.10
+target-version = "py310"
 
 # In addition to the standard set of exclusions, omit all tests, plus a specific file.
 extend-exclude = ["*/.*/",".*/","__pycache__","node_modules"]

requirements/common.in

@@ -1,7 +1,7 @@
 # Packages that are shared between deployment and dev environments.
 gunicorn==23.0.0
 whitenoise[brotli]==6.7.0  # Used by Whitenoise to provide Brotli-compressed versions of static files.
-Django==4.2.16
+Django==5.1.1
 celery==5.4.0  # celery needed for data ingestion
 cached-property==1.5.2 # needed for kombu with --require-hashes
 simplejson==3.19.3  # import simplejson
@@ -11,7 +11,7 @@ certifi==2024.8.30
 psycopg2-binary==2.9.9
 
 jsonschema==4.23.0  # import jsonschema
-djangorestframework==3.14.0  # Imported as rest_framework
+djangorestframework==3.15.2  # Imported as rest_framework
 django-cors-headers==4.4.0  # Listed as 3rd party app on settings.py
 mozlog==8.0.0
 

requirements/common.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.9
+# This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
 #    pip-compile --generate-hashes --output-file=requirements/common.txt requirements/common.in
@@ -359,9 +359,9 @@ crashtest==0.4.1 \
     --hash=sha256:80d7b1f316ebfbd429f648076d6275c877ba30ba48979de4191714a75266f0ce \
     --hash=sha256:8d23eac5fa660409f57472e3851dab7ac18aba459a8d19cbbba86d3d5aecd2a5
     # via cleo
-django==4.2.16 \
-    --hash=sha256:1ddc333a16fc139fd253035a1606bb24261951bbc3a6ca256717fa06cc41a898 \
-    --hash=sha256:6f1616c2786c408ce86ab7e10f792b8f15742f7b7b7460243929cb371e7f1dad
+django==5.1.1 \
+    --hash=sha256:021ffb7fdab3d2d388bc8c7c2434eb9c1f6f4d09e6119010bbb1694dda286bc2 \
+    --hash=sha256:71603f27dac22a6533fb38d83072eea9ddb4017fead6f67f2562a40402d61c3f
     # via
     #   -r requirements/common.in
     #   django-cors-headers
@@ -388,9 +388,9 @@ django-redis==5.4.0 \
     --hash=sha256:6a02abaa34b0fea8bf9b707d2c363ab6adc7409950b2db93602e6cb292818c42 \
     --hash=sha256:ebc88df7da810732e2af9987f7f426c96204bf89319df4c6da6ca9a2942edd5b
     # via -r requirements/common.in
-djangorestframework==3.14.0 \
-    --hash=sha256:579a333e6256b09489cbe0a067e66abe55c6595d8926be6b99423786334350c8 \
-    --hash=sha256:eb63f58c9f218e1a7d064d17a70751f528ed4e1d35547fdade9aaf4cd103fd08
+djangorestframework==3.15.2 \
+    --hash=sha256:2b8871b062ba1aefc2de01f773875441a961fefbf79f5eed1e32b2f096944b20 \
+    --hash=sha256:36fe88cd2d6c6bec23dca9804bab2ba5517a8bb9d8f47ebc68981b56840107ad
     # via -r requirements/common.in
 dockerflow==2024.4.2 \
     --hash=sha256:b9f92455449ba46555f57db34cccefc4c49d3533c67793624ab7e80a1625caa7 \
@@ -699,7 +699,9 @@ moz-measure-noise==2.60.1 \
 mozci[cache]==2.4.1 \
     --hash=sha256:389296a56e9f37f636c424b38092904334c6f767782098cad2a70ffab96f8f85 \
     --hash=sha256:c38cf0fde61424f327edfa9694f28c02536e6e0b49ddb23c43427829bedbde7f
-    # via -r requirements/common.in
+    # via
+    #   -r requirements/common.in
+    #   mozci
 mozfile==3.0.0 \
     --hash=sha256:3b0afcda2fa8b802ef657df80a56f21619008f61fcc14b756124028d7b7adf5c \
     --hash=sha256:92ca1a786abbdf5e6a7aada62d3a4e28f441ef069c7623223add45268e53c789
@@ -1022,14 +1024,12 @@ python-dateutil==2.9.0.post0 \
 python-jose[pycryptodome]==3.3.0 \
     --hash=sha256:55779b5e6ad599c6336191246e95eb2293a9ddebd555f796a65f838f07e5d78a \
     --hash=sha256:9b1376b023f8b298536eedd47ae1089bcdb848f1535ab30555cd92002d78923a
-    # via -r requirements/common.in
+    # via
+    #   -r requirements/common.in
+    #   python-jose
 python3-memcached==1.51 \
     --hash=sha256:7cbe5951d68eef69d948b7a7ed7decfbd101e15e7f5be007dcd1219ccc584859
     # via mozci
-pytz==2024.1 \
-    --hash=sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812 \
-    --hash=sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319
-    # via djangorestframework
 pyyaml==6.0.2 \
     --hash=sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff \
     --hash=sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48 \
@@ -1499,9 +1499,7 @@ types-python-dateutil==2.9.0.20240316 \
 typing-extensions==4.12.0 \
     --hash=sha256:8cbcdc8606ebcb0d95453ad7dc5065e6237b6aa230a31e81d0f440c30fed5fd8 \
     --hash=sha256:b349c66bea9016ac22978d800cfff206d5f9816951f12a7d0ec5578b0a819594
-    # via
-    #   asgiref
-    #   kombu
+    # via asgiref
 tzdata==2024.1 \
     --hash=sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd \
     --hash=sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252
@@ -1578,7 +1576,9 @@ wcwidth==0.2.13 \
 whitenoise[brotli]==6.7.0 \
     --hash=sha256:58c7a6cd811e275a6c91af22e96e87da0b1109e9a53bb7464116ef4c963bf636 \
     --hash=sha256:a1ae85e01fdc9815d12fa33f17765bc132ed2c54fa76daf9e39e879dd93566f6
-    # via -r requirements/common.in
+    # via
+    #   -r requirements/common.in
+    #   whitenoise
 yarl==1.9.4 \
     --hash=sha256:008d3e808d03ef28542372d01057fd09168419cdc8f848efe2804f894ae03e51 \
     --hash=sha256:03caa9507d3d3c83bca08650678e25364e1843b484f19986a527630ca376ecce \

requirements/dev.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.9
+# This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
 #    pip-compile --allow-unsafe --generate-hashes --output-file=requirements/dev.txt requirements/dev.in

setup.cfg

@@ -7,7 +7,7 @@ DJANGO_SETTINGS_MODULE=tests.settings
 addopts = -rsx -p no:mozlog
 # Make most warnings fatal (including the hidden by default DeprecationWarning):
 # https://docs.pytest.org/en/latest/warnings.html
-# https://docs.python.org/3.9/library/warnings.html#warning-categories
+# https://docs.python.org/3.10/library/warnings.html#warning-categories
 filterwarnings =
     error
     ignore::ImportWarning

tests/perf/auto_sheriffing_criteria/test_common_behaviour.py

@@ -2,7 +2,7 @@ from datetime import datetime, timedelta
 
 import pytest
 from django.conf import settings
-from typing import Callable
+from collections.abc import Callable
 
 from tests.perf.auto_sheriffing_criteria.conftest import CASSETTES_RECORDING_DATE
 from treeherder.config.settings import BZ_DATETIME_FORMAT

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py39
+envlist = py310
 isolated_build = true
 skipsdist=True
 

treeherder/etl/perf.py

@@ -2,7 +2,6 @@ import copy
 import logging
 from datetime import datetime
 from hashlib import sha1
-from typing import Optional
 
 import simplejson as json
 
@@ -119,7 +118,7 @@ def _test_should_alert_based_on(
 
 
 def _test_should_gather_replicates_based_on(
-    repository: Repository, suite_name: str, replicates: Optional[list] = None
+    repository: Repository, suite_name: str, replicates: list | None
 ) -> bool:
     """
     Determine if we should gather/ingest replicates. Currently, it's

treeherder/model/migrations/0001_squashed_0022_modify_bugscache_and_bugjobmap.py

@@ -925,23 +925,52 @@ class Migration(migrations.Migration):
             name='jobdetail',
             unique_together=set([('title', 'value', 'job')]),
         ),
-        migrations.AlterIndexTogether(
-            name='job',
-            index_together=set(
-                [
-                    ('repository', 'option_collection_hash', 'job_type', 'start_time'),
-                    ('repository', 'build_platform', 'job_type', 'start_time'),
-                    ('repository', 'submit_time'),
-                    ('machine_platform', 'option_collection_hash', 'push'),
-                    (
+        migrations.AddIndex(
+            model_name='job',
+            index=models.Index(
+                fields=['repository', 'option_collection_hash', 'job_type', 'start_time'],
+                name='job_reposit_63c897_idx',
+            ),
+        ),
+        migrations.AddIndex(
+            model_name='job',
+            index=models.Index(
+                fields=['repository', 'build_platform', 'job_type', 'start_time'],
+                name='job_reposit_fc1f71_idx',
+            ),
+        ),
+        migrations.AddIndex(
+            model_name='job',
+            index=models.Index(
+                fields=['repository', 'submit_time'],
+                name='job_reposit_2101af_idx',
+            ),
+        ),
+        migrations.AddIndex(
+            model_name='job',
+            index=models.Index(
+                fields=['machine_platform', 'option_collection_hash', 'push'],
+                name='job_machine_92bdd4_idx',
+            ),
+        ),
+        migrations.AddIndex(
+            model_name='job',
+            index=models.Index(
+                fields=[
                     'repository',
                     'build_platform',
                     'option_collection_hash',
                     'job_type',
                     'start_time',
+                ],
+                name='job_reposit_f4bb0f_idx'
+            )
         ),
-                    ('repository', 'job_type', 'start_time'),
-                ]
+        migrations.AddIndex(
+            model_name='job',
+            index=models.Index(
+                fields=['repository', 'job_type', 'start_time'],
+                name='job_reposit_f65dd6_idx',
             ),
         ),
         migrations.AlterUniqueTogether(
@@ -952,10 +981,6 @@ class Migration(migrations.Migration):
             name='failureline',
             unique_together=set([('job_log', 'line')]),
         ),
-        migrations.AlterIndexTogether(
-            name='failureline',
-            index_together=set([('job_guid', 'repository')]),
-        ),
         migrations.AlterUniqueTogether(
             name='commit',
             unique_together=set([('push', 'revision')]),
@@ -964,14 +989,11 @@ class Migration(migrations.Migration):
             name='bugjobmap',
             unique_together=set([('job', 'bug_id')]),
         ),
-        migrations.AlterIndexTogether(
-            name='failureline',
-            index_together=set(
-                [
-                    ('test', 'subtest', 'status', 'expected', 'created'),
-                    ('job_guid', 'repository'),
-                    ('signature', 'test', 'created'),
-                ]
+        migrations.AddIndex(
+            model_name='failureline',
+            index=models.Index(
+                fields=['job_guid', 'repository'],
+                name='failure_lin_job_gui_b67c6d_idx',
             ),
         ),
         migrations.AddIndex(

treeherder/model/migrations/0029_alter_failureline_index_together.py

@@ -15,8 +15,4 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.AlterIndexTogether(
-            name='failureline',
-            index_together={('job_guid', 'repository')},
-        ),
     ]

treeherder/perf/auto_perf_sheriffing/backfill_reports.py

@@ -1,7 +1,6 @@
 import logging
 from datetime import timedelta, datetime
 from itertools import zip_longest, groupby
-from typing import Optional
 
 import simplejson as json
 from django.db.models import QuerySet, Q, F
@@ -87,7 +86,7 @@ class AlertsPicker:
 
         def parent_or_sibling_from(
             alert_group: list[PerformanceAlert],
-        ) -> Optional[PerformanceAlert]:
+        ) -> PerformanceAlert | None:
             if len(alert_group) == 0:
                 return None
 

treeherder/perf/auto_perf_sheriffing/backfill_tool.py

@@ -1,7 +1,6 @@
 import logging
 
 from django.core.exceptions import ObjectDoesNotExist
-from typing import Union
 
 from treeherder.model.models import Job
 from treeherder.perf.exceptions import CannotBackfillError
@@ -14,7 +13,7 @@ class BackfillTool:
     def __init__(self, taskcluster_model: TaskclusterModel):
         self.__taskcluster = taskcluster_model
 
-    def backfill_job(self, job: Union[Job, str]) -> str:
+    def backfill_job(self, job: Job | str) -> str:
         if not isinstance(job, Job):
             job = self._fetch_job_by_id(job)
 

treeherder/perf/email.py

@@ -11,8 +11,6 @@ from dataclasses import dataclass, asdict
 from abc import ABC, abstractmethod
 import urllib.parse
 
-from typing import Union, Optional
-
 from django.conf import settings
 from treeherder.perf.models import (
     BackfillRecord,
@@ -40,7 +38,7 @@ class EmailWriter(ABC):
     def __init__(self):
         self._email = Email()
 
-    def prepare_new_email(self, must_mention: Union[list[object], object]) -> dict:
+    def prepare_new_email(self, must_mention: list[object] | object) -> dict:
         """
         Template method
         """
@@ -192,7 +190,7 @@ class BackfillReportContent:
         return f"{repository_name}:{previous_push}:{push}"
 
     @staticmethod
-    def __escape_markdown(text: str) -> Optional[str]:
+    def __escape_markdown(text: str) -> str | None:
         """
         Mostly copied "Example 2" from https://www.programcreek.com/python/?CodeExample=escape+markdown
         """

treeherder/perf/migrations/0001_squashed_0005_permit_github_links.py

@@ -326,10 +326,18 @@ class Migration(migrations.Migration):
             name='performancedatum',
             unique_together=set([('repository', 'job', 'push', 'signature')]),
         ),
-        migrations.AlterIndexTogether(
-            name='performancedatum',
-            index_together=set(
-                [('repository', 'signature', 'push_timestamp'), ('repository', 'signature', 'push')]
+        migrations.AddIndex(
+            model_name='performancedatum',
+            index=models.Index(
+                fields=['repository', 'signature', 'push_timestamp'],
+                name='performance_reposit_c9d328_idx',
+            ),
+        ),
+        migrations.AddIndex(
+            model_name='performancedatum',
+            index=models.Index(
+                fields=['repository', 'signature', 'push'],
+                name='performance_reposit_5486e2_idx',
             ),
         ),
         migrations.AlterUniqueTogether(

treeherder/perf/models.py

@@ -1,7 +1,6 @@
 import logging
 from datetime import datetime
 import json
-from typing import Optional
 from functools import reduce
 
 from django.contrib.auth.models import User
@@ -521,7 +520,7 @@ class PerformanceAlert(models.Model):
     manually_created = models.BooleanField(default=False)
 
     @property
-    def initial_culprit_job(self) -> Optional[Job]:
+    def initial_culprit_job(self) -> Job | None:
         if hasattr(self, "__initial_culprit_job"):
             return self.__initial_culprit_job
 
@@ -720,7 +719,7 @@ class BackfillRecord(models.Model):
         return self.alert.series_signature.platform
 
     @property
-    def job_symbol(self) -> Optional[str]:
+    def job_symbol(self) -> str | None:
         if not all([self.job_tier, self.job_group, self.job_type]):
             return None
 
@@ -845,7 +844,7 @@ class PerformanceSettings(models.Model):
         db_table = "performance_settings"
 
 
-def deepgetattr(obj: object, attr_chain: str) -> Optional[object]:
+def deepgetattr(obj: object, attr_chain: str) -> object | None:
     """Recursively follow an attribute chain to get the final value.
 
     @param attr_chain: e.g. 'repository.name', 'job_type', 'record.platform.architecture' etc

treeherder/perf/sheriffing_criteria/criteria_tracking.py

@@ -4,7 +4,6 @@ import logging
 from multiprocessing import cpu_count
 from multiprocessing.pool import Pool, ThreadPool, AsyncResult
 import time
-from typing import Union
 
 from datetime import datetime, timedelta
 
@@ -22,8 +21,8 @@ class CriteriaRecord:
     Framework: str
     Suite: str
     Test: str
-    EngineerTraction: Union[float, str]
-    FixRatio: Union[float, str]
+    EngineerTraction: float | str
+    FixRatio: float | str
     TotalAlerts: int
     LastUpdatedOn: datetime
     AllowSync: bool