Remove es7/elasticsearch7 references

Replace es7 and ES7 references with es and ES
Make name in docker version agnostic
Replace name in circle
This commit is contained in:
Smith Ellis 2022-10-28 15:20:59 +00:00
Родитель b0ae589881
Коммит 8e57e3164d
29 изменённых файлов: 89 добавлений и 89 удалений

Просмотреть файл

@ -30,10 +30,10 @@ jobs:
- run:
# copy synonym files to elasticsearch7 container, since circleci doesn't support volume mounts:
# https://circleci.com/docs/2.0/building-docker-images/#mounting-folders
name: Copy synonym files to elasticsearch7
name: Copy synonym files to elasticsearch
command: |
./bin/dc_ci.sh up -d elasticsearch7
docker cp ./kitsune/search/dictionaries/synonyms/. project-elasticsearch7-1:/usr/share/elasticsearch/config/synonyms
./bin/dc_ci.sh up -d elasticsearch
docker cp ./kitsune/search/dictionaries/synonyms/. project-elasticsearch-1:/usr/share/elasticsearch/config/synonyms
- run:
name: Run unit tests
command: ./bin/dc_ci.sh run test ./bin/run-unit-tests.sh

Просмотреть файл

@ -9,8 +9,8 @@ set -ex
urlwait
# wait for elasticsearch to be ready
urlwait http://elasticsearch7:9200 60
urlwait http://elasticsearch:9200 60
./manage.py es7_init --migrate-writes --migrate-reads
./manage.py es_init --migrate-writes --migrate-reads
./manage.py test --noinput --force-color --timing $@

Просмотреть файл

@ -13,7 +13,7 @@ services:
tty: true
depends_on:
- mariadb
- elasticsearch7
- elasticsearch
- kibana
- redis
- celery
@ -36,7 +36,7 @@ services:
volumes:
- mysqlvolume:/var/lib/mysql
elasticsearch7:
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.10.2
environment:
- discovery.type=single-node
@ -52,8 +52,8 @@ services:
ports:
- 5601:5601
environment:
ELASTICSEARCH_URL: http://elasticsearch7:9200
ELASTICSEARCH_HOSTS: http://elasticsearch7:9200
ELASTICSEARCH_URL: http://elasticsearch:9200
ELASTICSEARCH_HOSTS: http://elasticsearch:9200
redis:
image: redis:3

Просмотреть файл

@ -7,7 +7,7 @@ services:
env_file: .env-test
depends_on:
- mariadb
- elasticsearch7
- elasticsearch
- redis
prod:

Просмотреть файл

@ -165,7 +165,7 @@ or have entered data yourself through the admin interface.
2. Build the indicies
```
$ ./manage.py es7_init && ./manage.py es7_reindex
$ ./manage.py es_init && ./manage.py es_reindex
```
3. Now, exit from web's bash shell

Просмотреть файл

@ -12,11 +12,11 @@ To know whether a change you make to a Document will work in prod,
try it locally having already set up the mapping:
```
./manage.py es7_init --limit TestDocument
./manage.py es_init --limit TestDocument
... make changes to TestDocument ...
./manage.py es7_init --limit TestDocument
./manage.py es_init --limit TestDocument
```
If that fails with an error,
@ -26,16 +26,16 @@ and reindex everything into that index.
However if it succeeds then it should also work on prod.
Once the changes are deployed to prod,
and the mapping is updated with `es7_init`,
and the mapping is updated with `es_init`,
some documents may need to be reindexed.
This is because we disable dynamic mapping in `SumoDocument`,
to prevent a dynamic mapping of the wrong type being set up before `es7_init` was able to be run during a deployment.
to prevent a dynamic mapping of the wrong type being set up before `es_init` was able to be run during a deployment.
So to ensure no data is missing from the index,
run something like:
```
./manage.py es7_reindex --limit TestDocument --updated-after <datetime of deploy> --updated-before <datetime of mapping update>
./manage.py es_reindex --limit TestDocument --updated-after <datetime of deploy> --updated-before <datetime of mapping update>
```
### Indexing performance
@ -45,7 +45,7 @@ you might want to add the `--print-sql-count` argument when testing out your cha
to see how many SQL queries are being executed:
```sh
CELERY_TASK_ALWAYS_EAGER=True ./manage.py es7_reindex --print-sql-count --sql-chunk-size=100 --count=100
CELERY_TASK_ALWAYS_EAGER=True ./manage.py es_reindex --print-sql-count --sql-chunk-size=100 --count=100
```
If the result is much less than 100,
@ -87,7 +87,7 @@ as that returns a naive or aware datetime depending on the value of `USE_TZ`, wh
You can set the following variable in your .env file to enable the logging of the queries that are sent to your local ElasticSearch instance.
```
ES7_ENABLE_CONSOLE_LOGGING=True
ES_ENABLE_CONSOLE_LOGGING=True
```
### Simulate slow and out of order query responses
@ -234,7 +234,7 @@ In development synonyms can be updated very easily.
Save your changes in the text file and run:
```
./manage.py es7_init --reload-search-analyzers
./manage.py es_init --reload-search-analyzers
```
If no other changes were made to the index configurations,

Просмотреть файл

@ -74,7 +74,7 @@ app:
dms_fix_current_revisions: SECRET
dms_generate_missing_share_links: SECRET
dms_rebuild_kb: SECRET
dms_reindex_es7: SECRET
dms_reindex_es: SECRET
dms_reload_question_traffic_stats: SECRET
dms_reload_wiki_traffic_stats: SECRET
dms_report_employee_answers: SECRET

Просмотреть файл

@ -69,7 +69,7 @@ app:
dms_fix_current_revisions: SECRET
dms_generate_missing_share_links: SECRET
dms_rebuild_kb: SECRET
dms_reindex_es7: SECRET
dms_reindex_es: SECRET
dms_reload_question_traffic_stats: SECRET
dms_reload_wiki_traffic_stats: SECRET
dms_report_employee_answers: SECRET
@ -93,9 +93,9 @@ app:
engage_robots: True
es_index_prefix: "sumo_prod"
es_live_indexing: True
es7_use_ssl: True
es7_http_auth: SECRET
es7_cloud_id: SECRET
es_use_ssl: True
es_http_auth: SECRET
es_cloud_id: SECRET
ga_account: SECRET
ga_key: SECRET
ga_profile_id: SECRET

Просмотреть файл

@ -72,7 +72,7 @@ app:
dms_fix_current_revisions: SECRET
dms_generate_missing_share_links: SECRET
dms_rebuild_kb: SECRET
dms_reindex_es7: SECRET
dms_reindex_es: SECRET
dms_reload_question_traffic_stats: SECRET
dms_reload_wiki_traffic_stats: SECRET
dms_report_employee_answers: SECRET
@ -95,9 +95,9 @@ app:
engage_robots: False
es_index_prefix: "sumo_stage"
es_live_indexing: True
es7_use_ssl: True
es7_http_auth: SECRET
es7_cloud_id: SECRET
es_use_ssl: True
es_http_auth: SECRET
es_cloud_id: SECRET
ga_account: SECRET
ga_key: SECRET
ga_profile_id: SECRET

Просмотреть файл

@ -69,7 +69,7 @@ app:
dms_fix_current_revisions: SECRET
dms_generate_missing_share_links: SECRET
dms_rebuild_kb: SECRET
dms_reindex_es7: SECRET
dms_reindex_es: SECRET
dms_reload_question_traffic_stats: SECRET
dms_reload_wiki_traffic_stats: SECRET
dms_report_employee_answers: SECRET

Просмотреть файл

@ -73,7 +73,7 @@ app:
dms_fix_current_revisions: SECRET
dms_generate_missing_share_links: SECRET
dms_rebuild_kb: SECRET
dms_reindex_es7: SECRET
dms_reindex_es: SECRET
dms_reload_question_traffic_stats: SECRET
dms_reload_wiki_traffic_stats: SECRET
dms_report_employee_answers: SECRET
@ -99,9 +99,9 @@ app:
engage_robots: True
es_index_prefix: "sumo_prod"
es_live_indexing: True
es7_use_ssl: True
es7_http_auth: SECRET
es7_cloud_id: SECRET
es_use_ssl: True
es_http_auth: SECRET
es_cloud_id: SECRET
ga_account: SECRET
ga_key: SECRET
ga_profile_id: SECRET

Просмотреть файл

@ -72,7 +72,7 @@ app:
dms_fix_current_revisions: SECRET
dms_generate_missing_share_links: SECRET
dms_rebuild_kb: SECRET
dms_reindex_es7: SECRET
dms_reindex_es: SECRET
dms_reload_question_traffic_stats: SECRET
dms_reload_wiki_traffic_stats: SECRET
dms_report_employee_answers: SECRET
@ -95,9 +95,9 @@ app:
engage_robots: False
es_index_prefix: "sumo_stage"
es_live_indexing: True
es7_use_ssl: True
es7_http_auth: SECRET
es7_cloud_id: SECRET
es_use_ssl: True
es_http_auth: SECRET
es_cloud_id: SECRET
fxa_op_token_endpoint: SECRET
fxa_set_issuer: SECRET
fxa_op_authorization_endpoint: SECRET

Просмотреть файл

@ -6,7 +6,7 @@ from django.core.management.base import BaseCommand
from django.db import connection, transaction
from kitsune.questions.models import Answer, Question
from kitsune.search.es7_utils import index_objects_bulk
from kitsune.search.es_utils import index_objects_bulk
log = logging.getLogger("k.cron")

Просмотреть файл

@ -18,11 +18,11 @@ from elasticsearch_dsl.utils import AttrDict
from pyparsing import ParseException
from kitsune.search.config import (
DEFAULT_ES7_CONNECTION,
DEFAULT_ES7_REFRESH_INTERVAL,
DEFAULT_ES_CONNECTION,
DEFAULT_ES_REFRESH_INTERVAL,
UPDATE_RETRY_ON_CONFLICT,
)
from kitsune.search.es7_utils import es7_client
from kitsune.search.es_utils import es_client
from kitsune.search.parser import Parser
from kitsune.search.parser.tokens import TermToken
@ -40,7 +40,7 @@ class SumoDocument(DSLDocument):
"""Automatically set up each subclass' Index attribute."""
super().__init_subclass__(**kwargs)
cls.Index.using = DEFAULT_ES7_CONNECTION
cls.Index.using = DEFAULT_ES_CONNECTION
# this is here to ensure subclasses of subclasses of SumoDocument (e.g. AnswerDocument)
# use the same name in their index as their parent class (e.g. QuestionDocument) since
@ -55,7 +55,7 @@ class SumoDocument(DSLDocument):
cls.Index.read_alias = f"{cls.Index.base_name}_read"
cls.Index.write_alias = f"{cls.Index.base_name}_write"
# Bump the refresh interval to 1 minute
cls.Index.settings = {"refresh_interval": DEFAULT_ES7_REFRESH_INTERVAL}
cls.Index.settings = {"refresh_interval": DEFAULT_ES_REFRESH_INTERVAL}
# this is the attribute elastic-dsl actually uses to determine which index
# to query. we override the .search() method to get that to use the read
@ -88,7 +88,7 @@ class SumoDocument(DSLDocument):
@classmethod
def _update_alias(cls, alias, new_index):
client = es7_client()
client = es_client()
old_index = cls.alias_points_at(alias)
if not old_index:
client.indices.put_alias(new_index, alias)
@ -106,7 +106,7 @@ class SumoDocument(DSLDocument):
def alias_points_at(cls, alias):
"""Returns the index `alias` points at."""
try:
aliased_indices = list(es7_client().indices.get_alias(name=alias))
aliased_indices = list(es_client().indices.get_alias(name=alias))
except NotFoundError:
aliased_indices = []
@ -376,8 +376,8 @@ class SumoSearch(SumoSearchInterface):
"""Perform search, placing the results in `self.results`, and the total
number of results (across all pages) in `self.total`. Chainable."""
search = DSLSearch(using=es7_client(), index=self.get_index()).params(
**settings.ES7_SEARCH_PARAMS
search = DSLSearch(using=es_client(), index=self.get_index()).params(
**settings.ES_SEARCH_PARAMS
)
# add the search class' filter

Просмотреть файл

@ -371,6 +371,6 @@ ES_LOCALE_ANALYZERS = {
},
}
DEFAULT_ES7_CONNECTION = "es7_default"
DEFAULT_ES_CONNECTION = "es_default"
# default refresh_interval for all indices
DEFAULT_ES7_REFRESH_INTERVAL = "60s"
DEFAULT_ES_REFRESH_INTERVAL = "60s"

Просмотреть файл

@ -5,7 +5,7 @@ from kitsune.forums.models import Post
from kitsune.questions.models import Answer, Question
from kitsune.search import config
from kitsune.search.base import SumoDocument
from kitsune.search.es7_utils import es7_client
from kitsune.search.es_utils import es_client
from kitsune.search.fields import SumoLocaleAwareKeywordField, SumoLocaleAwareTextField
from kitsune.users.models import Profile
from kitsune.wiki import models as wiki_models
@ -16,7 +16,7 @@ from kitsune.wiki.config import (
TEMPLATES_CATEGORY,
)
connections.add_connection(config.DEFAULT_ES7_CONNECTION, es7_client())
connections.add_connection(config.DEFAULT_ES_CONNECTION, es_client())
class WikiDocument(SumoDocument):

Просмотреть файл

@ -4,7 +4,7 @@ import inspect
from celery import shared_task
from django.conf import settings
from elasticsearch import Elasticsearch
from elasticsearch.helpers import bulk as es7_bulk
from elasticsearch.helpers import bulk as es_bulk
from elasticsearch.helpers.errors import BulkIndexError
from elasticsearch_dsl import Document, UpdateByQuery, analyzer, char_filter, token_filter
@ -90,13 +90,13 @@ def es_analyzer_for_locale(locale, search_analyzer=False):
)
def es7_client(**kwargs):
"""Return an ES7 Elasticsearch client"""
def es_client(**kwargs):
"""Return an ES Elasticsearch client"""
# prefer a cloud_id if available
if es7_cloud_id := settings.ES7_CLOUD_ID:
kwargs.update({"cloud_id": es7_cloud_id, "http_auth": settings.ES7_HTTP_AUTH})
if es_cloud_id := settings.ES_CLOUD_ID:
kwargs.update({"cloud_id": es_cloud_id, "http_auth": settings.ES_HTTP_AUTH})
else:
kwargs.update({"hosts": settings.ES7_URLS})
kwargs.update({"hosts": settings.ES_URLS})
return Elasticsearch(**kwargs)
@ -166,8 +166,8 @@ def index_objects_bulk(
# if the request doesn't resolve within `timeout`,
# sleep for `timeout` then try again up to `settings.ES_BULK_MAX_RETRIES` times,
# before raising an exception:
success, errors = es7_bulk(
es7_client(
success, errors = es_bulk(
es_client(
timeout=timeout,
retry_on_timeout=True,
initial_backoff=timeout,
@ -198,7 +198,7 @@ def remove_from_field(doc_type_name, field_name, field_value):
f"}}"
)
update = UpdateByQuery(using=es7_client(), index=doc_type._index._name)
update = UpdateByQuery(using=es_client(), index=doc_type._index._name)
update = update.filter("term", **{field_name: field_value})
update = update.script(source=script, params={"value": field_value}, conflicts="proceed")

Просмотреть файл

@ -5,7 +5,7 @@ from elasticsearch_dsl.field import Keyword
from elasticsearch_dsl.field import Object as DSLObject
from elasticsearch_dsl.field import Text
from kitsune.search.es7_utils import es_analyzer_for_locale
from kitsune.search.es_utils import es_analyzer_for_locale
SUPPORTED_LANGUAGES = list(settings.SUMO_LANGUAGES)
# this is a test locale - no need to add it to ES

Просмотреть файл

@ -2,11 +2,11 @@ from django.core.management.base import BaseCommand
from elasticsearch_dsl.exceptions import IllegalOperation
from datetime import datetime, timezone
from kitsune.search.es7_utils import get_doc_types, es7_client
from kitsune.search.es_utils import get_doc_types, es_client
class Command(BaseCommand):
help = "Initialize ES7 document types"
help = "Initialize ES document types"
def add_arguments(self, parser):
parser.add_argument(
@ -34,7 +34,7 @@ class Command(BaseCommand):
)
def handle(self, *args, **kwargs):
client = es7_client()
client = es_client()
doc_types = get_doc_types()
limit = kwargs["limit"]

Просмотреть файл

@ -5,11 +5,11 @@ from django.core.management.base import BaseCommand
from django.db import connection, reset_queries
from dateutil.parser import parse as dateutil_parse
from kitsune.search.es7_utils import get_doc_types, index_objects_bulk
from kitsune.search.es_utils import get_doc_types, index_objects_bulk
class Command(BaseCommand):
help = "Reindex ES7 documents"
help = "Reindex ES documents"
def add_arguments(self, parser):
parser.add_argument(

Просмотреть файл

@ -1,5 +1,5 @@
from django.db.models.signals import post_save, post_delete
from kitsune.search.es7_utils import index_object, delete_object, index_objects_bulk
from kitsune.search.es_utils import index_object, delete_object, index_objects_bulk
from kitsune.search.decorators import search_receiver
from kitsune.forums.models import Thread, Post

Просмотреть файл

@ -1,5 +1,5 @@
from django.db.models.signals import post_save, post_delete, m2m_changed
from kitsune.search.es7_utils import (
from kitsune.search.es_utils import (
index_object,
delete_object,
index_objects_bulk,

Просмотреть файл

@ -1,7 +1,7 @@
from django.db.models.signals import post_save, post_delete, m2m_changed
from django.contrib.auth.models import User, Group
from kitsune.users.models import Profile
from kitsune.search.es7_utils import (
from kitsune.search.es_utils import (
index_object,
delete_object,
remove_from_field,

Просмотреть файл

@ -2,7 +2,7 @@ from django.db.models.signals import m2m_changed, post_delete, post_save
from kitsune.products.models import Product, Topic
from kitsune.search.decorators import search_receiver
from kitsune.search.es7_utils import delete_object, index_object, remove_from_field
from kitsune.search.es_utils import delete_object, index_object, remove_from_field
from kitsune.wiki.models import Document

Просмотреть файл

@ -1,6 +1,6 @@
from django.test.utils import override_settings
from kitsune.search.es7_utils import get_doc_types
from kitsune.search.es_utils import get_doc_types
from kitsune.sumo.tests import TestCase

Просмотреть файл

@ -2,8 +2,8 @@ from django.test.utils import override_settings
from kitsune.search.tests import Elastic7TestCase
from kitsune.search.documents import ProfileDocument
from kitsune.users.tests import ProfileFactory, GroupFactory
from elasticsearch.helpers import bulk as es7_bulk
from kitsune.search.es7_utils import es7_client
from elasticsearch.helpers import bulk as es_bulk
from kitsune.search.es_utils import es_client
@override_settings(ES_LIVE_INDEXING=False)
@ -28,7 +28,7 @@ class ToActionTests(Elastic7TestCase):
def test_index_bulk_empty_list(self):
payload = self.prepare().to_action("index", is_bulk=True)
es7_bulk(es7_client(), [payload])
es_bulk(es_client(), [payload])
self.assertEqual(self.doc.group_ids, [])
def test_update_empty_list(self):
@ -37,5 +37,5 @@ class ToActionTests(Elastic7TestCase):
def test_update_bulk_empty_list(self):
payload = self.prepare().to_action("update", is_bulk=True)
es7_bulk(es7_client(), [payload])
es_bulk(es_client(), [payload])
self.assertEqual(self.doc.group_ids, [])

Просмотреть файл

@ -1,7 +1,7 @@
from kitsune.search.tests import Elastic7TestCase
from kitsune.questions.tests import QuestionFactory
from django.test.utils import override_settings
from kitsune.search.es7_utils import index_objects_bulk
from kitsune.search.es_utils import index_objects_bulk
from elasticsearch.helpers.errors import BulkIndexError
from elasticsearch.exceptions import NotFoundError
from kitsune.search.documents import QuestionDocument

Просмотреть файл

@ -738,14 +738,14 @@ CSRF_COOKIE_SECURE = config("CSRF_COOKIE_SECURE", default=not DEBUG, cast=bool)
#
# Connection information for Elastic 7
ES_TIMEOUT = 5 # Timeout for querying requests
ES7_URLS = config("ES7_URLS", cast=Csv(), default="elasticsearch7:9200")
ES7_CLOUD_ID = config("ES7_CLOUD_ID", default="")
ES7_USE_SSL = config("ES7_USE_SSL", default=False, cast=bool)
ES7_HTTP_AUTH = config("ES7_HTTP_AUTH", default="", cast=Csv())
ES7_ENABLE_CONSOLE_LOGGING = config("ES7_ENABLE_CONSOLE_LOGGING", default=False, cast=bool)
# Pass parameters to the ES7 client
ES_URLS = config("ES_URLS", cast=Csv(), default="elasticsearch:9200")
ES_CLOUD_ID = config("ES_CLOUD_ID", default="")
ES_USE_SSL = config("ES_USE_SSL", default=False, cast=bool)
ES_HTTP_AUTH = config("ES_HTTP_AUTH", default="", cast=Csv())
ES_ENABLE_CONSOLE_LOGGING = config("ES_ENABLE_CONSOLE_LOGGING", default=False, cast=bool)
# Pass parameters to the ES client
# like "search_type": "dfs_query_then_fetch"
ES7_SEARCH_PARAMS = {"request_timeout": ES_TIMEOUT}
ES_SEARCH_PARAMS = {"request_timeout": ES_TIMEOUT}
# This is prepended to index names to get the final read/write index
# names used by kitsune. This is so that you can have multiple
@ -1059,7 +1059,7 @@ DMS_UPDATE_SEARCH_CTR_METRIC = config("DMS_UPDATE_SEARCH_CTR_METRIC", default=No
DMS_UPDATE_CONTRIBUTOR_METRICS = config("DMS_UPDATE_CONTRIBUTOR_METRICS", default=None)
DMS_AUTO_ARCHIVE_OLD_QUESTIONS = config("DMS_AUTO_ARCHIVE_OLD_QUESTIONS", default=None)
DMS_REINDEX = config("DMS_REINDEX", default=None)
DMS_REINDEX_ES7 = config("DMS_REINDEX_ES7", default=None)
DMS_REINDEX_ES = config("DMS_REINDEX_ES", default=None)
# DMS_PROCESS_EXIT_SURVEYS = config("DMS_PROCESS_EXIT_SURVEYS", default=None)
# DMS_SURVEY_RECENT_ASKERS = config("DMS_SURVEY_RECENT_ASKERS", default=None)
# DMS_UPDATE_VISITORS_METRIC = config('DMS_UPDATE_VISITORS_METRIC', default=None)
@ -1123,7 +1123,7 @@ TOLL_FREE_REGEX = re.compile(r"^.*8(00|33|44|55|66|77|88)[2-9]\d{6,}$")
REGEX_TIMEOUT = config("REGEX_TIMEOUT", default=5, cast=int)
NANP_REGEX = re.compile(r"[0-9]{3}-?[a-zA-Z2-9][a-zA-Z0-9]{2}-?[a-zA-Z0-9]{4}")
if ES7_ENABLE_CONSOLE_LOGGING and DEV:
if ES_ENABLE_CONSOLE_LOGGING and DEV:
es_trace_logger = logging.getLogger("elasticsearch.trace")
es_trace_logger.setLevel(logging.INFO)
handler = logging.StreamHandler()

Просмотреть файл

@ -85,11 +85,11 @@ def job_send_welcome_emails():
# only run on readonly clusters, where no signals will be triggered:
skip=(not settings.READ_ONLY),
)
@babis.decorator(ping_after=settings.DMS_REINDEX_ES7)
def job_reindex_es7():
# Index items newer than 90 minutes old in ES7
@babis.decorator(ping_after=settings.DMS_REINDEX_ES)
def job_reindex_es():
# Index items newer than 90 minutes old in ES
after = (timezone.now() - datetime.timedelta(minutes=90)).isoformat()
call_command("es7_reindex --updated-after {}".format(after))
call_command("es_reindex --updated-after {}".format(after))
# Every 6 hours.