add es7 reindexing cronjob for read only clusters (#4709)

https://github.com/mozilla/sumo-project/issues/697

k8s/regions/frankfurt/dev.yaml

@@ -70,6 +70,7 @@ app:
     dms_process_exit_surveys: SECRET
     dms_rebuild_kb: SECRET
     dms_reindex: SECRET
+    dms_reindex_es7: SECRET
     dms_reindex_users_that_contributed_yesterday: SECRET
     dms_reload_question_traffic_stats: SECRET
     dms_reload_wiki_traffic_stats: SECRET

k8s/regions/frankfurt/prod.yaml

@@ -68,6 +68,7 @@ app:
     dms_process_exit_surveys: SECRET
     dms_rebuild_kb: SECRET
     dms_reindex: SECRET
+    dms_reindex_es7: SECRET
     dms_reindex_users_that_contributed_yesterday: SECRET
     dms_reload_question_traffic_stats: SECRET
     dms_reload_wiki_traffic_stats: SECRET

k8s/regions/frankfurt/stage.yaml

@@ -71,6 +71,7 @@ app:
     dms_process_exit_surveys: SECRET
     dms_rebuild_kb: SECRET
     dms_reindex: SECRET
+    dms_reindex_es7: SECRET
     dms_reindex_users_that_contributed_yesterday: SECRET
     dms_reload_question_traffic_stats: SECRET
     dms_reload_wiki_traffic_stats: SECRET

k8s/regions/frankfurt/test.yaml

@@ -70,6 +70,7 @@ app:
     dms_process_exit_surveys: SECRET
     dms_rebuild_kb: SECRET
     dms_reindex: SECRET
+    dms_reindex_es7: SECRET
     dms_reindex_users_that_contributed_yesterday: SECRET
     dms_reload_question_traffic_stats: SECRET
     dms_reload_wiki_traffic_stats: SECRET

k8s/regions/oregon-a/prod.yaml

@@ -72,6 +72,7 @@ app:
     dms_process_exit_surveys: SECRET
     dms_rebuild_kb: SECRET
     dms_reindex: SECRET
+    dms_reindex_es7: SECRET
     dms_reindex_users_that_contributed_yesterday: SECRET
     dms_reload_question_traffic_stats: SECRET
     dms_reload_wiki_traffic_stats: SECRET

k8s/regions/oregon-a/stage.yaml

@@ -71,6 +71,7 @@ app:
     dms_process_exit_surveys: SECRET
     dms_rebuild_kb: SECRET
     dms_reindex: SECRET
+    dms_reindex_es7: SECRET
     dms_reindex_users_that_contributed_yesterday: SECRET
     dms_reload_question_traffic_stats: SECRET
     dms_reload_wiki_traffic_stats: SECRET

k8s/regions/oregon-b/prod.yaml

@@ -72,6 +72,7 @@ app:
     dms_process_exit_surveys: SECRET
     dms_rebuild_kb: SECRET
     dms_reindex: SECRET
+    dms_reindex_es7: SECRET
     dms_reindex_users_that_contributed_yesterday: SECRET
     dms_reload_question_traffic_stats: SECRET
     dms_reload_wiki_traffic_stats: SECRET

k8s/regions/oregon-b/stage.yaml

@@ -71,6 +71,7 @@ app:
     dms_process_exit_surveys: SECRET
     dms_rebuild_kb: SECRET
     dms_reindex: SECRET
+    dms_reindex_es7: SECRET
     dms_reindex_users_that_contributed_yesterday: SECRET
     dms_reload_question_traffic_stats: SECRET
     dms_reload_wiki_traffic_stats: SECRET

kitsune/settings.py

@@ -1164,6 +1164,7 @@ DMS_UPDATE_SEARCH_CTR_METRIC = config("DMS_UPDATE_SEARCH_CTR_METRIC", default=No
 DMS_UPDATE_CONTRIBUTOR_METRICS = config("DMS_UPDATE_CONTRIBUTOR_METRICS", default=None)
 DMS_AUTO_ARCHIVE_OLD_QUESTIONS = config("DMS_AUTO_ARCHIVE_OLD_QUESTIONS", default=None)
 DMS_REINDEX = config("DMS_REINDEX", default=None)
+DMS_REINDEX_ES7 = config("DMS_REINDEX_ES7", default=None)
 DMS_PROCESS_EXIT_SURVEYS = config("DMS_PROCESS_EXIT_SURVEYS", default=None)
 DMS_SURVEY_RECENT_ASKERS = config("DMS_SURVEY_RECENT_ASKERS", default=None)
 # DMS_UPDATE_VISITORS_METRIC = config('DMS_UPDATE_VISITORS_METRIC', default=None)

scripts/cron.py

@@ -6,6 +6,7 @@ from subprocess import check_call
 import babis
 from apscheduler.schedulers.blocking import BlockingScheduler
 from django.conf import settings
+from django.utils import timezone
 
 MANAGE = os.path.join(settings.ROOT, "manage.py")
 schedule = BlockingScheduler()
@@ -96,6 +97,24 @@ def job_reindex():
     call_command("esreindex --minutes-ago 90")
 
 
+@scheduled_job(
+    "cron",
+    month="*",
+    day="*",
+    hour="*",
+    minute="45",
+    max_instances=1,
+    coalesce=True,
+    # only run on readonly clusters, where no signals will be triggered:
+    skip=(not settings.READ_ONLY),
+)
+@babis.decorator(ping_after=settings.DMS_REINDEX_ES7)
+def job_reindex_es7():
+    # Index items newer than 90 minutes old in ES7
+    after = (timezone.now() - datetime.timedelta(minutes=90)).isoformat()
+    call_command("es7_reindex --updated-after {}".format(after))
+
+
 # Every 6 hours.
 @scheduled_job(
     "cron",