From 580c2210bf9815cf4610f5b8cd9022069b5c5efe Mon Sep 17 00:00:00 2001 From: Leo McArdle Date: Thu, 11 Mar 2021 11:01:43 +0000 Subject: [PATCH] add es7 reindexing cronjob for read only clusters (#4709) https://github.com/mozilla/sumo-project/issues/697 --- k8s/regions/frankfurt/dev.yaml | 1 + k8s/regions/frankfurt/prod.yaml | 1 + k8s/regions/frankfurt/stage.yaml | 1 + k8s/regions/frankfurt/test.yaml | 1 + k8s/regions/oregon-a/prod.yaml | 1 + k8s/regions/oregon-a/stage.yaml | 1 + k8s/regions/oregon-b/prod.yaml | 1 + k8s/regions/oregon-b/stage.yaml | 1 + kitsune/settings.py | 1 + scripts/cron.py | 19 +++++++++++++++++++ 10 files changed, 28 insertions(+) diff --git a/k8s/regions/frankfurt/dev.yaml b/k8s/regions/frankfurt/dev.yaml index 5b31e466b..45928cb45 100644 --- a/k8s/regions/frankfurt/dev.yaml +++ b/k8s/regions/frankfurt/dev.yaml @@ -70,6 +70,7 @@ app: dms_process_exit_surveys: SECRET dms_rebuild_kb: SECRET dms_reindex: SECRET + dms_reindex_es7: SECRET dms_reindex_users_that_contributed_yesterday: SECRET dms_reload_question_traffic_stats: SECRET dms_reload_wiki_traffic_stats: SECRET diff --git a/k8s/regions/frankfurt/prod.yaml b/k8s/regions/frankfurt/prod.yaml index ece49dab2..c184b8794 100644 --- a/k8s/regions/frankfurt/prod.yaml +++ b/k8s/regions/frankfurt/prod.yaml @@ -68,6 +68,7 @@ app: dms_process_exit_surveys: SECRET dms_rebuild_kb: SECRET dms_reindex: SECRET + dms_reindex_es7: SECRET dms_reindex_users_that_contributed_yesterday: SECRET dms_reload_question_traffic_stats: SECRET dms_reload_wiki_traffic_stats: SECRET diff --git a/k8s/regions/frankfurt/stage.yaml b/k8s/regions/frankfurt/stage.yaml index 9dc00b3bc..ff5d0ba9e 100644 --- a/k8s/regions/frankfurt/stage.yaml +++ b/k8s/regions/frankfurt/stage.yaml @@ -71,6 +71,7 @@ app: dms_process_exit_surveys: SECRET dms_rebuild_kb: SECRET dms_reindex: SECRET + dms_reindex_es7: SECRET dms_reindex_users_that_contributed_yesterday: SECRET dms_reload_question_traffic_stats: SECRET dms_reload_wiki_traffic_stats: SECRET diff --git a/k8s/regions/frankfurt/test.yaml b/k8s/regions/frankfurt/test.yaml index 805c7a2b5..cdffa9398 100644 --- a/k8s/regions/frankfurt/test.yaml +++ b/k8s/regions/frankfurt/test.yaml @@ -70,6 +70,7 @@ app: dms_process_exit_surveys: SECRET dms_rebuild_kb: SECRET dms_reindex: SECRET + dms_reindex_es7: SECRET dms_reindex_users_that_contributed_yesterday: SECRET dms_reload_question_traffic_stats: SECRET dms_reload_wiki_traffic_stats: SECRET diff --git a/k8s/regions/oregon-a/prod.yaml b/k8s/regions/oregon-a/prod.yaml index 2b055e0e3..ee27176bd 100644 --- a/k8s/regions/oregon-a/prod.yaml +++ b/k8s/regions/oregon-a/prod.yaml @@ -72,6 +72,7 @@ app: dms_process_exit_surveys: SECRET dms_rebuild_kb: SECRET dms_reindex: SECRET + dms_reindex_es7: SECRET dms_reindex_users_that_contributed_yesterday: SECRET dms_reload_question_traffic_stats: SECRET dms_reload_wiki_traffic_stats: SECRET diff --git a/k8s/regions/oregon-a/stage.yaml b/k8s/regions/oregon-a/stage.yaml index 724138c3a..d778c1bff 100644 --- a/k8s/regions/oregon-a/stage.yaml +++ b/k8s/regions/oregon-a/stage.yaml @@ -71,6 +71,7 @@ app: dms_process_exit_surveys: SECRET dms_rebuild_kb: SECRET dms_reindex: SECRET + dms_reindex_es7: SECRET dms_reindex_users_that_contributed_yesterday: SECRET dms_reload_question_traffic_stats: SECRET dms_reload_wiki_traffic_stats: SECRET diff --git a/k8s/regions/oregon-b/prod.yaml b/k8s/regions/oregon-b/prod.yaml index ca37ab604..62caeca67 100644 --- a/k8s/regions/oregon-b/prod.yaml +++ b/k8s/regions/oregon-b/prod.yaml @@ -72,6 +72,7 @@ app: dms_process_exit_surveys: SECRET dms_rebuild_kb: SECRET dms_reindex: SECRET + dms_reindex_es7: SECRET dms_reindex_users_that_contributed_yesterday: SECRET dms_reload_question_traffic_stats: SECRET dms_reload_wiki_traffic_stats: SECRET diff --git a/k8s/regions/oregon-b/stage.yaml b/k8s/regions/oregon-b/stage.yaml index f96787fab..7128ccba2 100644 --- a/k8s/regions/oregon-b/stage.yaml +++ b/k8s/regions/oregon-b/stage.yaml @@ -71,6 +71,7 @@ app: dms_process_exit_surveys: SECRET dms_rebuild_kb: SECRET dms_reindex: SECRET + dms_reindex_es7: SECRET dms_reindex_users_that_contributed_yesterday: SECRET dms_reload_question_traffic_stats: SECRET dms_reload_wiki_traffic_stats: SECRET diff --git a/kitsune/settings.py b/kitsune/settings.py index 1a271837c..17f8ae259 100644 --- a/kitsune/settings.py +++ b/kitsune/settings.py @@ -1164,6 +1164,7 @@ DMS_UPDATE_SEARCH_CTR_METRIC = config("DMS_UPDATE_SEARCH_CTR_METRIC", default=No DMS_UPDATE_CONTRIBUTOR_METRICS = config("DMS_UPDATE_CONTRIBUTOR_METRICS", default=None) DMS_AUTO_ARCHIVE_OLD_QUESTIONS = config("DMS_AUTO_ARCHIVE_OLD_QUESTIONS", default=None) DMS_REINDEX = config("DMS_REINDEX", default=None) +DMS_REINDEX_ES7 = config("DMS_REINDEX_ES7", default=None) DMS_PROCESS_EXIT_SURVEYS = config("DMS_PROCESS_EXIT_SURVEYS", default=None) DMS_SURVEY_RECENT_ASKERS = config("DMS_SURVEY_RECENT_ASKERS", default=None) # DMS_UPDATE_VISITORS_METRIC = config('DMS_UPDATE_VISITORS_METRIC', default=None) diff --git a/scripts/cron.py b/scripts/cron.py index 022bb7fdd..9be443b3f 100644 --- a/scripts/cron.py +++ b/scripts/cron.py @@ -6,6 +6,7 @@ from subprocess import check_call import babis from apscheduler.schedulers.blocking import BlockingScheduler from django.conf import settings +from django.utils import timezone MANAGE = os.path.join(settings.ROOT, "manage.py") schedule = BlockingScheduler() @@ -96,6 +97,24 @@ def job_reindex(): call_command("esreindex --minutes-ago 90") +@scheduled_job( + "cron", + month="*", + day="*", + hour="*", + minute="45", + max_instances=1, + coalesce=True, + # only run on readonly clusters, where no signals will be triggered: + skip=(not settings.READ_ONLY), +) +@babis.decorator(ping_after=settings.DMS_REINDEX_ES7) +def job_reindex_es7(): + # Index items newer than 90 minutes old in ES7 + after = (timezone.now() - datetime.timedelta(minutes=90)).isoformat() + call_command("es7_reindex --updated-after {}".format(after)) + + # Every 6 hours. @scheduled_job( "cron",