Bug 1851253 - store new failure lines in database for persistent storage across reboots.

1310f34128

initialize_data.sh

@@ -12,4 +12,6 @@ if [ "${DATABASE_URL}" == "psql://postgres:mozilla1234@postgres:5432/treeherder"
     ./manage.py load_initial_data
 fi
 
+./manage.py createcachetable
+
 exec "$@"

tests/test_setup.py

@@ -1,7 +1,7 @@
 import pytest
 import responses
 from celery import current_app
-from django.core.cache import cache
+from django.core.cache import caches
 from django.core.management import call_command
 
 from treeherder.utils.http import fetch_text
@@ -26,12 +26,18 @@ def test_no_missing_migrations():
     call_command("makemigrations", interactive=False, dry_run=True, check_changes=True)
 
 
+@pytest.mark.django_db
 def test_django_cache():
     """Test the Django cache backend & associated server are properly set up."""
     k, v = "my_key", "my_value"
+    cache = caches["default"]
     cache.set(k, v, 10)
     assert cache.get(k) == v
 
+    db_cache = caches["db_cache"]
+    db_cache.set(k, v, 10)
+    assert db_cache.get(k) == v
+
 
 @current_app.task
 def add(x, y):

treeherder/config/settings.py

@@ -162,6 +162,10 @@ CACHES = {
             "SOCKET_CONNECT_TIMEOUT": 5,
         },
     },
+    "db_cache": {
+        "BACKEND": "django.core.cache.backends.db.DatabaseCache",
+        "LOCATION": "new_failure_cache",
+    },
 }
 
 # Internationalization

treeherder/model/error_summary.py

@@ -3,7 +3,7 @@ import logging
 import re
 import newrelic.agent
 
-from django.core.cache import cache
+from django.core.cache import caches
 
 from treeherder.model.models import Bugscache, TextLogError
 
@@ -15,6 +15,8 @@ logger = logging.getLogger(__name__)
 BUG_SUGGESTION_CACHE_TIMEOUT = 86400
 LINE_CACHE_TIMEOUT_DAYS = 21
 LINE_CACHE_TIMEOUT = 86400 * LINE_CACHE_TIMEOUT_DAYS
+db_cache = caches["db_cache"]
+cache = caches["default"]
 
 LEAK_RE = re.compile(r"\d+ bytes leaked \((.+)\)$|leak at (.+)$")
 CRASH_RE = re.compile(r".+ application crashed \[@ (.+)\] \|.+")
@@ -42,7 +44,7 @@ def get_error_summary(job, queryset=None):
     line_cache_key = "mc_error_lines"
     if job.repository == "comm-central":
         line_cache_key = "cc_error_lines"
-    line_cache = cache.get(line_cache_key)
+    line_cache = db_cache.get(line_cache_key)
     if line_cache is None:
         line_cache = {str(job.submit_time.date()): {}}
     else:
@@ -84,7 +86,7 @@ def get_error_summary(job, queryset=None):
         logger.error("error caching error_summary for job %s: %s", job.id, e, exc_info=True)
 
     try:
-        cache.set(line_cache_key, line_cache, LINE_CACHE_TIMEOUT)
+        db_cache.set(line_cache_key, line_cache, LINE_CACHE_TIMEOUT)
     except Exception as e:
         newrelic.agent.record_custom_event("error caching error_lines for job", job.id)
         logger.error("error caching error_lines for job %s: %s", job.id, e, exc_info=True)

treeherder/webapp/api/note.py

@@ -1,4 +1,4 @@
-from django.core.cache import cache
+from django.core.cache import caches
 
 from rest_framework import viewsets
 from rest_framework.decorators import action
@@ -92,8 +92,9 @@ class NoteViewSet(viewsets.ViewSet):
 
         if fc_id == 2:  # this is for fixed_by_commit (backout | follow_up_commit)
             # remove cached failure line counts
+            db_cache = caches["db_cache"]
             line_cache_key = "error_lines"
-            line_cache = cache.get(line_cache_key)
+            line_cache = db_cache.get(line_cache_key)
             date = current_job.submit_time.date().isoformat()
             if line_cache and date in line_cache.keys():
                 for err in TextLogError.objects.filter(job=current_job):
@@ -107,7 +108,7 @@ class NoteViewSet(viewsets.ViewSet):
                         ):
                             del line_cache[date]["new_lines"][cache_clean_line]
                         try:
-                            cache.set(line_cache_key, line_cache, LINE_CACHE_TIMEOUT)
+                            db_cache.set(line_cache_key, line_cache, LINE_CACHE_TIMEOUT)
                         except Exception as e:
                             logger.error(
                                 "error caching error_lines for job %s: %s",