From 8e57e3164d71021a620e129babd7e3ea0df87953 Mon Sep 17 00:00:00 2001 From: Smith Ellis Date: Fri, 28 Oct 2022 15:20:59 +0000 Subject: [PATCH] Remove es7/elasticsearch7 references Replace es7 and ES7 references with es and ES Make name in docker version agnostic Replace name in circle --- .circleci/config.yml | 6 +++--- bin/run-unit-tests.sh | 4 ++-- docker-compose.yml | 8 ++++---- docker/docker-compose.ci.yml | 2 +- docs/hacking_howto.md | 2 +- docs/search.md | 16 ++++++++-------- k8s/regions/frankfurt/dev.yaml | 2 +- k8s/regions/frankfurt/prod.yaml | 8 ++++---- k8s/regions/frankfurt/stage.yaml | 8 ++++---- k8s/regions/frankfurt/test.yaml | 2 +- k8s/regions/oregon/prod.yaml | 8 ++++---- k8s/regions/oregon/stage.yaml | 8 ++++---- .../commands/auto_archive_old_questions.py | 2 +- kitsune/search/base.py | 18 +++++++++--------- kitsune/search/config.py | 4 ++-- kitsune/search/documents.py | 4 ++-- kitsune/search/{es7_utils.py => es_utils.py} | 18 +++++++++--------- kitsune/search/fields.py | 2 +- .../commands/{es7_init.py => es_init.py} | 6 +++--- .../commands/{es7_reindex.py => es_reindex.py} | 4 ++-- kitsune/search/signals/forums.py | 2 +- kitsune/search/signals/questions.py | 2 +- kitsune/search/signals/users.py | 2 +- kitsune/search/signals/wiki.py | 2 +- kitsune/search/tests/__init__.py | 2 +- kitsune/search/tests/test_base.py | 8 ++++---- kitsune/search/tests/test_es7_utils.py | 2 +- kitsune/settings.py | 18 +++++++++--------- scripts/cron.py | 8 ++++---- 29 files changed, 89 insertions(+), 89 deletions(-) rename kitsune/search/{es7_utils.py => es_utils.py} (94%) rename kitsune/search/management/commands/{es7_init.py => es_init.py} (95%) rename kitsune/search/management/commands/{es7_reindex.py => es_reindex.py} (97%) diff --git a/.circleci/config.yml b/.circleci/config.yml index ccc8e62ec..f39c6c84c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,10 +30,10 @@ jobs: - run: # copy synonym files to elasticsearch7 container, since circleci doesn't support volume mounts: # https://circleci.com/docs/2.0/building-docker-images/#mounting-folders - name: Copy synonym files to elasticsearch7 + name: Copy synonym files to elasticsearch command: | - ./bin/dc_ci.sh up -d elasticsearch7 - docker cp ./kitsune/search/dictionaries/synonyms/. project-elasticsearch7-1:/usr/share/elasticsearch/config/synonyms + ./bin/dc_ci.sh up -d elasticsearch + docker cp ./kitsune/search/dictionaries/synonyms/. project-elasticsearch-1:/usr/share/elasticsearch/config/synonyms - run: name: Run unit tests command: ./bin/dc_ci.sh run test ./bin/run-unit-tests.sh diff --git a/bin/run-unit-tests.sh b/bin/run-unit-tests.sh index d98897a1a..4b46f5cba 100755 --- a/bin/run-unit-tests.sh +++ b/bin/run-unit-tests.sh @@ -9,8 +9,8 @@ set -ex urlwait # wait for elasticsearch to be ready -urlwait http://elasticsearch7:9200 60 +urlwait http://elasticsearch:9200 60 -./manage.py es7_init --migrate-writes --migrate-reads +./manage.py es_init --migrate-writes --migrate-reads ./manage.py test --noinput --force-color --timing $@ diff --git a/docker-compose.yml b/docker-compose.yml index cbb3f3dcc..4093b6f9e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,7 @@ services: tty: true depends_on: - mariadb - - elasticsearch7 + - elasticsearch - kibana - redis - celery @@ -36,7 +36,7 @@ services: volumes: - mysqlvolume:/var/lib/mysql - elasticsearch7: + elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:7.10.2 environment: - discovery.type=single-node @@ -52,8 +52,8 @@ services: ports: - 5601:5601 environment: - ELASTICSEARCH_URL: http://elasticsearch7:9200 - ELASTICSEARCH_HOSTS: http://elasticsearch7:9200 + ELASTICSEARCH_URL: http://elasticsearch:9200 + ELASTICSEARCH_HOSTS: http://elasticsearch:9200 redis: image: redis:3 diff --git a/docker/docker-compose.ci.yml b/docker/docker-compose.ci.yml index 08e8c8819..4268eb0c5 100644 --- a/docker/docker-compose.ci.yml +++ b/docker/docker-compose.ci.yml @@ -7,7 +7,7 @@ services: env_file: .env-test depends_on: - mariadb - - elasticsearch7 + - elasticsearch - redis prod: diff --git a/docs/hacking_howto.md b/docs/hacking_howto.md index e999990c9..82d8fbe6f 100644 --- a/docs/hacking_howto.md +++ b/docs/hacking_howto.md @@ -165,7 +165,7 @@ or have entered data yourself through the admin interface. 2. Build the indicies ``` - $ ./manage.py es7_init && ./manage.py es7_reindex + $ ./manage.py es_init && ./manage.py es_reindex ``` 3. Now, exit from web's bash shell diff --git a/docs/search.md b/docs/search.md index 88373780b..3306d19ce 100644 --- a/docs/search.md +++ b/docs/search.md @@ -12,11 +12,11 @@ To know whether a change you make to a Document will work in prod, try it locally having already set up the mapping: ``` -./manage.py es7_init --limit TestDocument +./manage.py es_init --limit TestDocument ... make changes to TestDocument ... -./manage.py es7_init --limit TestDocument +./manage.py es_init --limit TestDocument ``` If that fails with an error, @@ -26,16 +26,16 @@ and reindex everything into that index. However if it succeeds then it should also work on prod. Once the changes are deployed to prod, -and the mapping is updated with `es7_init`, +and the mapping is updated with `es_init`, some documents may need to be reindexed. This is because we disable dynamic mapping in `SumoDocument`, -to prevent a dynamic mapping of the wrong type being set up before `es7_init` was able to be run during a deployment. +to prevent a dynamic mapping of the wrong type being set up before `es_init` was able to be run during a deployment. So to ensure no data is missing from the index, run something like: ``` -./manage.py es7_reindex --limit TestDocument --updated-after --updated-before +./manage.py es_reindex --limit TestDocument --updated-after --updated-before ``` ### Indexing performance @@ -45,7 +45,7 @@ you might want to add the `--print-sql-count` argument when testing out your cha to see how many SQL queries are being executed: ```sh -CELERY_TASK_ALWAYS_EAGER=True ./manage.py es7_reindex --print-sql-count --sql-chunk-size=100 --count=100 +CELERY_TASK_ALWAYS_EAGER=True ./manage.py es_reindex --print-sql-count --sql-chunk-size=100 --count=100 ``` If the result is much less than 100, @@ -87,7 +87,7 @@ as that returns a naive or aware datetime depending on the value of `USE_TZ`, wh You can set the following variable in your .env file to enable the logging of the queries that are sent to your local ElasticSearch instance. ``` -ES7_ENABLE_CONSOLE_LOGGING=True +ES_ENABLE_CONSOLE_LOGGING=True ``` ### Simulate slow and out of order query responses @@ -234,7 +234,7 @@ In development synonyms can be updated very easily. Save your changes in the text file and run: ``` -./manage.py es7_init --reload-search-analyzers +./manage.py es_init --reload-search-analyzers ``` If no other changes were made to the index configurations, diff --git a/k8s/regions/frankfurt/dev.yaml b/k8s/regions/frankfurt/dev.yaml index 33e64d033..c22e3e59e 100644 --- a/k8s/regions/frankfurt/dev.yaml +++ b/k8s/regions/frankfurt/dev.yaml @@ -74,7 +74,7 @@ app: dms_fix_current_revisions: SECRET dms_generate_missing_share_links: SECRET dms_rebuild_kb: SECRET - dms_reindex_es7: SECRET + dms_reindex_es: SECRET dms_reload_question_traffic_stats: SECRET dms_reload_wiki_traffic_stats: SECRET dms_report_employee_answers: SECRET diff --git a/k8s/regions/frankfurt/prod.yaml b/k8s/regions/frankfurt/prod.yaml index 8afb4c9ef..cfdb4ad2c 100644 --- a/k8s/regions/frankfurt/prod.yaml +++ b/k8s/regions/frankfurt/prod.yaml @@ -69,7 +69,7 @@ app: dms_fix_current_revisions: SECRET dms_generate_missing_share_links: SECRET dms_rebuild_kb: SECRET - dms_reindex_es7: SECRET + dms_reindex_es: SECRET dms_reload_question_traffic_stats: SECRET dms_reload_wiki_traffic_stats: SECRET dms_report_employee_answers: SECRET @@ -93,9 +93,9 @@ app: engage_robots: True es_index_prefix: "sumo_prod" es_live_indexing: True - es7_use_ssl: True - es7_http_auth: SECRET - es7_cloud_id: SECRET + es_use_ssl: True + es_http_auth: SECRET + es_cloud_id: SECRET ga_account: SECRET ga_key: SECRET ga_profile_id: SECRET diff --git a/k8s/regions/frankfurt/stage.yaml b/k8s/regions/frankfurt/stage.yaml index 5fa093f14..08936f98d 100644 --- a/k8s/regions/frankfurt/stage.yaml +++ b/k8s/regions/frankfurt/stage.yaml @@ -72,7 +72,7 @@ app: dms_fix_current_revisions: SECRET dms_generate_missing_share_links: SECRET dms_rebuild_kb: SECRET - dms_reindex_es7: SECRET + dms_reindex_es: SECRET dms_reload_question_traffic_stats: SECRET dms_reload_wiki_traffic_stats: SECRET dms_report_employee_answers: SECRET @@ -95,9 +95,9 @@ app: engage_robots: False es_index_prefix: "sumo_stage" es_live_indexing: True - es7_use_ssl: True - es7_http_auth: SECRET - es7_cloud_id: SECRET + es_use_ssl: True + es_http_auth: SECRET + es_cloud_id: SECRET ga_account: SECRET ga_key: SECRET ga_profile_id: SECRET diff --git a/k8s/regions/frankfurt/test.yaml b/k8s/regions/frankfurt/test.yaml index 008fbddf9..57d92e23f 100644 --- a/k8s/regions/frankfurt/test.yaml +++ b/k8s/regions/frankfurt/test.yaml @@ -69,7 +69,7 @@ app: dms_fix_current_revisions: SECRET dms_generate_missing_share_links: SECRET dms_rebuild_kb: SECRET - dms_reindex_es7: SECRET + dms_reindex_es: SECRET dms_reload_question_traffic_stats: SECRET dms_reload_wiki_traffic_stats: SECRET dms_report_employee_answers: SECRET diff --git a/k8s/regions/oregon/prod.yaml b/k8s/regions/oregon/prod.yaml index c4681e93f..6b2aa630a 100644 --- a/k8s/regions/oregon/prod.yaml +++ b/k8s/regions/oregon/prod.yaml @@ -73,7 +73,7 @@ app: dms_fix_current_revisions: SECRET dms_generate_missing_share_links: SECRET dms_rebuild_kb: SECRET - dms_reindex_es7: SECRET + dms_reindex_es: SECRET dms_reload_question_traffic_stats: SECRET dms_reload_wiki_traffic_stats: SECRET dms_report_employee_answers: SECRET @@ -99,9 +99,9 @@ app: engage_robots: True es_index_prefix: "sumo_prod" es_live_indexing: True - es7_use_ssl: True - es7_http_auth: SECRET - es7_cloud_id: SECRET + es_use_ssl: True + es_http_auth: SECRET + es_cloud_id: SECRET ga_account: SECRET ga_key: SECRET ga_profile_id: SECRET diff --git a/k8s/regions/oregon/stage.yaml b/k8s/regions/oregon/stage.yaml index b01da3b2f..76ed97311 100644 --- a/k8s/regions/oregon/stage.yaml +++ b/k8s/regions/oregon/stage.yaml @@ -72,7 +72,7 @@ app: dms_fix_current_revisions: SECRET dms_generate_missing_share_links: SECRET dms_rebuild_kb: SECRET - dms_reindex_es7: SECRET + dms_reindex_es: SECRET dms_reload_question_traffic_stats: SECRET dms_reload_wiki_traffic_stats: SECRET dms_report_employee_answers: SECRET @@ -95,9 +95,9 @@ app: engage_robots: False es_index_prefix: "sumo_stage" es_live_indexing: True - es7_use_ssl: True - es7_http_auth: SECRET - es7_cloud_id: SECRET + es_use_ssl: True + es_http_auth: SECRET + es_cloud_id: SECRET fxa_op_token_endpoint: SECRET fxa_set_issuer: SECRET fxa_op_authorization_endpoint: SECRET diff --git a/kitsune/questions/management/commands/auto_archive_old_questions.py b/kitsune/questions/management/commands/auto_archive_old_questions.py index 550f7b926..4f882783d 100644 --- a/kitsune/questions/management/commands/auto_archive_old_questions.py +++ b/kitsune/questions/management/commands/auto_archive_old_questions.py @@ -6,7 +6,7 @@ from django.core.management.base import BaseCommand from django.db import connection, transaction from kitsune.questions.models import Answer, Question -from kitsune.search.es7_utils import index_objects_bulk +from kitsune.search.es_utils import index_objects_bulk log = logging.getLogger("k.cron") diff --git a/kitsune/search/base.py b/kitsune/search/base.py index c39956f4b..5039c57a1 100644 --- a/kitsune/search/base.py +++ b/kitsune/search/base.py @@ -18,11 +18,11 @@ from elasticsearch_dsl.utils import AttrDict from pyparsing import ParseException from kitsune.search.config import ( - DEFAULT_ES7_CONNECTION, - DEFAULT_ES7_REFRESH_INTERVAL, + DEFAULT_ES_CONNECTION, + DEFAULT_ES_REFRESH_INTERVAL, UPDATE_RETRY_ON_CONFLICT, ) -from kitsune.search.es7_utils import es7_client +from kitsune.search.es_utils import es_client from kitsune.search.parser import Parser from kitsune.search.parser.tokens import TermToken @@ -40,7 +40,7 @@ class SumoDocument(DSLDocument): """Automatically set up each subclass' Index attribute.""" super().__init_subclass__(**kwargs) - cls.Index.using = DEFAULT_ES7_CONNECTION + cls.Index.using = DEFAULT_ES_CONNECTION # this is here to ensure subclasses of subclasses of SumoDocument (e.g. AnswerDocument) # use the same name in their index as their parent class (e.g. QuestionDocument) since @@ -55,7 +55,7 @@ class SumoDocument(DSLDocument): cls.Index.read_alias = f"{cls.Index.base_name}_read" cls.Index.write_alias = f"{cls.Index.base_name}_write" # Bump the refresh interval to 1 minute - cls.Index.settings = {"refresh_interval": DEFAULT_ES7_REFRESH_INTERVAL} + cls.Index.settings = {"refresh_interval": DEFAULT_ES_REFRESH_INTERVAL} # this is the attribute elastic-dsl actually uses to determine which index # to query. we override the .search() method to get that to use the read @@ -88,7 +88,7 @@ class SumoDocument(DSLDocument): @classmethod def _update_alias(cls, alias, new_index): - client = es7_client() + client = es_client() old_index = cls.alias_points_at(alias) if not old_index: client.indices.put_alias(new_index, alias) @@ -106,7 +106,7 @@ class SumoDocument(DSLDocument): def alias_points_at(cls, alias): """Returns the index `alias` points at.""" try: - aliased_indices = list(es7_client().indices.get_alias(name=alias)) + aliased_indices = list(es_client().indices.get_alias(name=alias)) except NotFoundError: aliased_indices = [] @@ -376,8 +376,8 @@ class SumoSearch(SumoSearchInterface): """Perform search, placing the results in `self.results`, and the total number of results (across all pages) in `self.total`. Chainable.""" - search = DSLSearch(using=es7_client(), index=self.get_index()).params( - **settings.ES7_SEARCH_PARAMS + search = DSLSearch(using=es_client(), index=self.get_index()).params( + **settings.ES_SEARCH_PARAMS ) # add the search class' filter diff --git a/kitsune/search/config.py b/kitsune/search/config.py index c7b172ef5..ab553187c 100644 --- a/kitsune/search/config.py +++ b/kitsune/search/config.py @@ -371,6 +371,6 @@ ES_LOCALE_ANALYZERS = { }, } -DEFAULT_ES7_CONNECTION = "es7_default" +DEFAULT_ES_CONNECTION = "es_default" # default refresh_interval for all indices -DEFAULT_ES7_REFRESH_INTERVAL = "60s" +DEFAULT_ES_REFRESH_INTERVAL = "60s" diff --git a/kitsune/search/documents.py b/kitsune/search/documents.py index 705889fb1..6d4cb84ff 100644 --- a/kitsune/search/documents.py +++ b/kitsune/search/documents.py @@ -5,7 +5,7 @@ from kitsune.forums.models import Post from kitsune.questions.models import Answer, Question from kitsune.search import config from kitsune.search.base import SumoDocument -from kitsune.search.es7_utils import es7_client +from kitsune.search.es_utils import es_client from kitsune.search.fields import SumoLocaleAwareKeywordField, SumoLocaleAwareTextField from kitsune.users.models import Profile from kitsune.wiki import models as wiki_models @@ -16,7 +16,7 @@ from kitsune.wiki.config import ( TEMPLATES_CATEGORY, ) -connections.add_connection(config.DEFAULT_ES7_CONNECTION, es7_client()) +connections.add_connection(config.DEFAULT_ES_CONNECTION, es_client()) class WikiDocument(SumoDocument): diff --git a/kitsune/search/es7_utils.py b/kitsune/search/es_utils.py similarity index 94% rename from kitsune/search/es7_utils.py rename to kitsune/search/es_utils.py index d0e077f97..8dd8ff4ee 100644 --- a/kitsune/search/es7_utils.py +++ b/kitsune/search/es_utils.py @@ -4,7 +4,7 @@ import inspect from celery import shared_task from django.conf import settings from elasticsearch import Elasticsearch -from elasticsearch.helpers import bulk as es7_bulk +from elasticsearch.helpers import bulk as es_bulk from elasticsearch.helpers.errors import BulkIndexError from elasticsearch_dsl import Document, UpdateByQuery, analyzer, char_filter, token_filter @@ -90,13 +90,13 @@ def es_analyzer_for_locale(locale, search_analyzer=False): ) -def es7_client(**kwargs): - """Return an ES7 Elasticsearch client""" +def es_client(**kwargs): + """Return an ES Elasticsearch client""" # prefer a cloud_id if available - if es7_cloud_id := settings.ES7_CLOUD_ID: - kwargs.update({"cloud_id": es7_cloud_id, "http_auth": settings.ES7_HTTP_AUTH}) + if es_cloud_id := settings.ES_CLOUD_ID: + kwargs.update({"cloud_id": es_cloud_id, "http_auth": settings.ES_HTTP_AUTH}) else: - kwargs.update({"hosts": settings.ES7_URLS}) + kwargs.update({"hosts": settings.ES_URLS}) return Elasticsearch(**kwargs) @@ -166,8 +166,8 @@ def index_objects_bulk( # if the request doesn't resolve within `timeout`, # sleep for `timeout` then try again up to `settings.ES_BULK_MAX_RETRIES` times, # before raising an exception: - success, errors = es7_bulk( - es7_client( + success, errors = es_bulk( + es_client( timeout=timeout, retry_on_timeout=True, initial_backoff=timeout, @@ -198,7 +198,7 @@ def remove_from_field(doc_type_name, field_name, field_value): f"}}" ) - update = UpdateByQuery(using=es7_client(), index=doc_type._index._name) + update = UpdateByQuery(using=es_client(), index=doc_type._index._name) update = update.filter("term", **{field_name: field_value}) update = update.script(source=script, params={"value": field_value}, conflicts="proceed") diff --git a/kitsune/search/fields.py b/kitsune/search/fields.py index 616ab2230..3b87176b9 100644 --- a/kitsune/search/fields.py +++ b/kitsune/search/fields.py @@ -5,7 +5,7 @@ from elasticsearch_dsl.field import Keyword from elasticsearch_dsl.field import Object as DSLObject from elasticsearch_dsl.field import Text -from kitsune.search.es7_utils import es_analyzer_for_locale +from kitsune.search.es_utils import es_analyzer_for_locale SUPPORTED_LANGUAGES = list(settings.SUMO_LANGUAGES) # this is a test locale - no need to add it to ES diff --git a/kitsune/search/management/commands/es7_init.py b/kitsune/search/management/commands/es_init.py similarity index 95% rename from kitsune/search/management/commands/es7_init.py rename to kitsune/search/management/commands/es_init.py index ee8e4fb16..92321ef47 100644 --- a/kitsune/search/management/commands/es7_init.py +++ b/kitsune/search/management/commands/es_init.py @@ -2,11 +2,11 @@ from django.core.management.base import BaseCommand from elasticsearch_dsl.exceptions import IllegalOperation from datetime import datetime, timezone -from kitsune.search.es7_utils import get_doc_types, es7_client +from kitsune.search.es_utils import get_doc_types, es_client class Command(BaseCommand): - help = "Initialize ES7 document types" + help = "Initialize ES document types" def add_arguments(self, parser): parser.add_argument( @@ -34,7 +34,7 @@ class Command(BaseCommand): ) def handle(self, *args, **kwargs): - client = es7_client() + client = es_client() doc_types = get_doc_types() limit = kwargs["limit"] diff --git a/kitsune/search/management/commands/es7_reindex.py b/kitsune/search/management/commands/es_reindex.py similarity index 97% rename from kitsune/search/management/commands/es7_reindex.py rename to kitsune/search/management/commands/es_reindex.py index cb2e437ee..cf1c4c2ad 100644 --- a/kitsune/search/management/commands/es7_reindex.py +++ b/kitsune/search/management/commands/es_reindex.py @@ -5,11 +5,11 @@ from django.core.management.base import BaseCommand from django.db import connection, reset_queries from dateutil.parser import parse as dateutil_parse -from kitsune.search.es7_utils import get_doc_types, index_objects_bulk +from kitsune.search.es_utils import get_doc_types, index_objects_bulk class Command(BaseCommand): - help = "Reindex ES7 documents" + help = "Reindex ES documents" def add_arguments(self, parser): parser.add_argument( diff --git a/kitsune/search/signals/forums.py b/kitsune/search/signals/forums.py index ad6548894..fbd7c4b1f 100644 --- a/kitsune/search/signals/forums.py +++ b/kitsune/search/signals/forums.py @@ -1,5 +1,5 @@ from django.db.models.signals import post_save, post_delete -from kitsune.search.es7_utils import index_object, delete_object, index_objects_bulk +from kitsune.search.es_utils import index_object, delete_object, index_objects_bulk from kitsune.search.decorators import search_receiver from kitsune.forums.models import Thread, Post diff --git a/kitsune/search/signals/questions.py b/kitsune/search/signals/questions.py index 7f871a86d..b3d55f930 100644 --- a/kitsune/search/signals/questions.py +++ b/kitsune/search/signals/questions.py @@ -1,5 +1,5 @@ from django.db.models.signals import post_save, post_delete, m2m_changed -from kitsune.search.es7_utils import ( +from kitsune.search.es_utils import ( index_object, delete_object, index_objects_bulk, diff --git a/kitsune/search/signals/users.py b/kitsune/search/signals/users.py index 345051982..aad278bea 100644 --- a/kitsune/search/signals/users.py +++ b/kitsune/search/signals/users.py @@ -1,7 +1,7 @@ from django.db.models.signals import post_save, post_delete, m2m_changed from django.contrib.auth.models import User, Group from kitsune.users.models import Profile -from kitsune.search.es7_utils import ( +from kitsune.search.es_utils import ( index_object, delete_object, remove_from_field, diff --git a/kitsune/search/signals/wiki.py b/kitsune/search/signals/wiki.py index 0dbaad52f..722e96f21 100644 --- a/kitsune/search/signals/wiki.py +++ b/kitsune/search/signals/wiki.py @@ -2,7 +2,7 @@ from django.db.models.signals import m2m_changed, post_delete, post_save from kitsune.products.models import Product, Topic from kitsune.search.decorators import search_receiver -from kitsune.search.es7_utils import delete_object, index_object, remove_from_field +from kitsune.search.es_utils import delete_object, index_object, remove_from_field from kitsune.wiki.models import Document diff --git a/kitsune/search/tests/__init__.py b/kitsune/search/tests/__init__.py index 0dc770e4a..bbfd236c2 100644 --- a/kitsune/search/tests/__init__.py +++ b/kitsune/search/tests/__init__.py @@ -1,6 +1,6 @@ from django.test.utils import override_settings -from kitsune.search.es7_utils import get_doc_types +from kitsune.search.es_utils import get_doc_types from kitsune.sumo.tests import TestCase diff --git a/kitsune/search/tests/test_base.py b/kitsune/search/tests/test_base.py index 42c282602..34ee4f388 100644 --- a/kitsune/search/tests/test_base.py +++ b/kitsune/search/tests/test_base.py @@ -2,8 +2,8 @@ from django.test.utils import override_settings from kitsune.search.tests import Elastic7TestCase from kitsune.search.documents import ProfileDocument from kitsune.users.tests import ProfileFactory, GroupFactory -from elasticsearch.helpers import bulk as es7_bulk -from kitsune.search.es7_utils import es7_client +from elasticsearch.helpers import bulk as es_bulk +from kitsune.search.es_utils import es_client @override_settings(ES_LIVE_INDEXING=False) @@ -28,7 +28,7 @@ class ToActionTests(Elastic7TestCase): def test_index_bulk_empty_list(self): payload = self.prepare().to_action("index", is_bulk=True) - es7_bulk(es7_client(), [payload]) + es_bulk(es_client(), [payload]) self.assertEqual(self.doc.group_ids, []) def test_update_empty_list(self): @@ -37,5 +37,5 @@ class ToActionTests(Elastic7TestCase): def test_update_bulk_empty_list(self): payload = self.prepare().to_action("update", is_bulk=True) - es7_bulk(es7_client(), [payload]) + es_bulk(es_client(), [payload]) self.assertEqual(self.doc.group_ids, []) diff --git a/kitsune/search/tests/test_es7_utils.py b/kitsune/search/tests/test_es7_utils.py index 98c426128..ac181ba83 100644 --- a/kitsune/search/tests/test_es7_utils.py +++ b/kitsune/search/tests/test_es7_utils.py @@ -1,7 +1,7 @@ from kitsune.search.tests import Elastic7TestCase from kitsune.questions.tests import QuestionFactory from django.test.utils import override_settings -from kitsune.search.es7_utils import index_objects_bulk +from kitsune.search.es_utils import index_objects_bulk from elasticsearch.helpers.errors import BulkIndexError from elasticsearch.exceptions import NotFoundError from kitsune.search.documents import QuestionDocument diff --git a/kitsune/settings.py b/kitsune/settings.py index b6558ecd4..db7951ebf 100644 --- a/kitsune/settings.py +++ b/kitsune/settings.py @@ -738,14 +738,14 @@ CSRF_COOKIE_SECURE = config("CSRF_COOKIE_SECURE", default=not DEBUG, cast=bool) # # Connection information for Elastic 7 ES_TIMEOUT = 5 # Timeout for querying requests -ES7_URLS = config("ES7_URLS", cast=Csv(), default="elasticsearch7:9200") -ES7_CLOUD_ID = config("ES7_CLOUD_ID", default="") -ES7_USE_SSL = config("ES7_USE_SSL", default=False, cast=bool) -ES7_HTTP_AUTH = config("ES7_HTTP_AUTH", default="", cast=Csv()) -ES7_ENABLE_CONSOLE_LOGGING = config("ES7_ENABLE_CONSOLE_LOGGING", default=False, cast=bool) -# Pass parameters to the ES7 client +ES_URLS = config("ES_URLS", cast=Csv(), default="elasticsearch:9200") +ES_CLOUD_ID = config("ES_CLOUD_ID", default="") +ES_USE_SSL = config("ES_USE_SSL", default=False, cast=bool) +ES_HTTP_AUTH = config("ES_HTTP_AUTH", default="", cast=Csv()) +ES_ENABLE_CONSOLE_LOGGING = config("ES_ENABLE_CONSOLE_LOGGING", default=False, cast=bool) +# Pass parameters to the ES client # like "search_type": "dfs_query_then_fetch" -ES7_SEARCH_PARAMS = {"request_timeout": ES_TIMEOUT} +ES_SEARCH_PARAMS = {"request_timeout": ES_TIMEOUT} # This is prepended to index names to get the final read/write index # names used by kitsune. This is so that you can have multiple @@ -1059,7 +1059,7 @@ DMS_UPDATE_SEARCH_CTR_METRIC = config("DMS_UPDATE_SEARCH_CTR_METRIC", default=No DMS_UPDATE_CONTRIBUTOR_METRICS = config("DMS_UPDATE_CONTRIBUTOR_METRICS", default=None) DMS_AUTO_ARCHIVE_OLD_QUESTIONS = config("DMS_AUTO_ARCHIVE_OLD_QUESTIONS", default=None) DMS_REINDEX = config("DMS_REINDEX", default=None) -DMS_REINDEX_ES7 = config("DMS_REINDEX_ES7", default=None) +DMS_REINDEX_ES = config("DMS_REINDEX_ES", default=None) # DMS_PROCESS_EXIT_SURVEYS = config("DMS_PROCESS_EXIT_SURVEYS", default=None) # DMS_SURVEY_RECENT_ASKERS = config("DMS_SURVEY_RECENT_ASKERS", default=None) # DMS_UPDATE_VISITORS_METRIC = config('DMS_UPDATE_VISITORS_METRIC', default=None) @@ -1123,7 +1123,7 @@ TOLL_FREE_REGEX = re.compile(r"^.*8(00|33|44|55|66|77|88)[2-9]\d{6,}$") REGEX_TIMEOUT = config("REGEX_TIMEOUT", default=5, cast=int) NANP_REGEX = re.compile(r"[0-9]{3}-?[a-zA-Z2-9][a-zA-Z0-9]{2}-?[a-zA-Z0-9]{4}") -if ES7_ENABLE_CONSOLE_LOGGING and DEV: +if ES_ENABLE_CONSOLE_LOGGING and DEV: es_trace_logger = logging.getLogger("elasticsearch.trace") es_trace_logger.setLevel(logging.INFO) handler = logging.StreamHandler() diff --git a/scripts/cron.py b/scripts/cron.py index 0a241517a..00c13f485 100644 --- a/scripts/cron.py +++ b/scripts/cron.py @@ -85,11 +85,11 @@ def job_send_welcome_emails(): # only run on readonly clusters, where no signals will be triggered: skip=(not settings.READ_ONLY), ) -@babis.decorator(ping_after=settings.DMS_REINDEX_ES7) -def job_reindex_es7(): - # Index items newer than 90 minutes old in ES7 +@babis.decorator(ping_after=settings.DMS_REINDEX_ES) +def job_reindex_es(): + # Index items newer than 90 minutes old in ES after = (timezone.now() - datetime.timedelta(minutes=90)).isoformat() - call_command("es7_reindex --updated-after {}".format(after)) + call_command("es_reindex --updated-after {}".format(after)) # Every 6 hours.