зеркало из https://github.com/mozilla/kitsune.git
Remove es7/elasticsearch7 references
Replace es7 and ES7 references with es and ES Make name in docker version agnostic Replace name in circle
This commit is contained in:
Родитель
b0ae589881
Коммит
8e57e3164d
|
@ -30,10 +30,10 @@ jobs:
|
||||||
- run:
|
- run:
|
||||||
# copy synonym files to elasticsearch7 container, since circleci doesn't support volume mounts:
|
# copy synonym files to elasticsearch7 container, since circleci doesn't support volume mounts:
|
||||||
# https://circleci.com/docs/2.0/building-docker-images/#mounting-folders
|
# https://circleci.com/docs/2.0/building-docker-images/#mounting-folders
|
||||||
name: Copy synonym files to elasticsearch7
|
name: Copy synonym files to elasticsearch
|
||||||
command: |
|
command: |
|
||||||
./bin/dc_ci.sh up -d elasticsearch7
|
./bin/dc_ci.sh up -d elasticsearch
|
||||||
docker cp ./kitsune/search/dictionaries/synonyms/. project-elasticsearch7-1:/usr/share/elasticsearch/config/synonyms
|
docker cp ./kitsune/search/dictionaries/synonyms/. project-elasticsearch-1:/usr/share/elasticsearch/config/synonyms
|
||||||
- run:
|
- run:
|
||||||
name: Run unit tests
|
name: Run unit tests
|
||||||
command: ./bin/dc_ci.sh run test ./bin/run-unit-tests.sh
|
command: ./bin/dc_ci.sh run test ./bin/run-unit-tests.sh
|
||||||
|
|
|
@ -9,8 +9,8 @@ set -ex
|
||||||
urlwait
|
urlwait
|
||||||
|
|
||||||
# wait for elasticsearch to be ready
|
# wait for elasticsearch to be ready
|
||||||
urlwait http://elasticsearch7:9200 60
|
urlwait http://elasticsearch:9200 60
|
||||||
|
|
||||||
./manage.py es7_init --migrate-writes --migrate-reads
|
./manage.py es_init --migrate-writes --migrate-reads
|
||||||
|
|
||||||
./manage.py test --noinput --force-color --timing $@
|
./manage.py test --noinput --force-color --timing $@
|
||||||
|
|
|
@ -13,7 +13,7 @@ services:
|
||||||
tty: true
|
tty: true
|
||||||
depends_on:
|
depends_on:
|
||||||
- mariadb
|
- mariadb
|
||||||
- elasticsearch7
|
- elasticsearch
|
||||||
- kibana
|
- kibana
|
||||||
- redis
|
- redis
|
||||||
- celery
|
- celery
|
||||||
|
@ -36,7 +36,7 @@ services:
|
||||||
volumes:
|
volumes:
|
||||||
- mysqlvolume:/var/lib/mysql
|
- mysqlvolume:/var/lib/mysql
|
||||||
|
|
||||||
elasticsearch7:
|
elasticsearch:
|
||||||
image: docker.elastic.co/elasticsearch/elasticsearch:7.10.2
|
image: docker.elastic.co/elasticsearch/elasticsearch:7.10.2
|
||||||
environment:
|
environment:
|
||||||
- discovery.type=single-node
|
- discovery.type=single-node
|
||||||
|
@ -52,8 +52,8 @@ services:
|
||||||
ports:
|
ports:
|
||||||
- 5601:5601
|
- 5601:5601
|
||||||
environment:
|
environment:
|
||||||
ELASTICSEARCH_URL: http://elasticsearch7:9200
|
ELASTICSEARCH_URL: http://elasticsearch:9200
|
||||||
ELASTICSEARCH_HOSTS: http://elasticsearch7:9200
|
ELASTICSEARCH_HOSTS: http://elasticsearch:9200
|
||||||
|
|
||||||
redis:
|
redis:
|
||||||
image: redis:3
|
image: redis:3
|
||||||
|
|
|
@ -7,7 +7,7 @@ services:
|
||||||
env_file: .env-test
|
env_file: .env-test
|
||||||
depends_on:
|
depends_on:
|
||||||
- mariadb
|
- mariadb
|
||||||
- elasticsearch7
|
- elasticsearch
|
||||||
- redis
|
- redis
|
||||||
|
|
||||||
prod:
|
prod:
|
||||||
|
|
|
@ -165,7 +165,7 @@ or have entered data yourself through the admin interface.
|
||||||
2. Build the indicies
|
2. Build the indicies
|
||||||
|
|
||||||
```
|
```
|
||||||
$ ./manage.py es7_init && ./manage.py es7_reindex
|
$ ./manage.py es_init && ./manage.py es_reindex
|
||||||
```
|
```
|
||||||
|
|
||||||
3. Now, exit from web's bash shell
|
3. Now, exit from web's bash shell
|
||||||
|
|
|
@ -12,11 +12,11 @@ To know whether a change you make to a Document will work in prod,
|
||||||
try it locally having already set up the mapping:
|
try it locally having already set up the mapping:
|
||||||
|
|
||||||
```
|
```
|
||||||
./manage.py es7_init --limit TestDocument
|
./manage.py es_init --limit TestDocument
|
||||||
|
|
||||||
... make changes to TestDocument ...
|
... make changes to TestDocument ...
|
||||||
|
|
||||||
./manage.py es7_init --limit TestDocument
|
./manage.py es_init --limit TestDocument
|
||||||
```
|
```
|
||||||
|
|
||||||
If that fails with an error,
|
If that fails with an error,
|
||||||
|
@ -26,16 +26,16 @@ and reindex everything into that index.
|
||||||
However if it succeeds then it should also work on prod.
|
However if it succeeds then it should also work on prod.
|
||||||
|
|
||||||
Once the changes are deployed to prod,
|
Once the changes are deployed to prod,
|
||||||
and the mapping is updated with `es7_init`,
|
and the mapping is updated with `es_init`,
|
||||||
some documents may need to be reindexed.
|
some documents may need to be reindexed.
|
||||||
This is because we disable dynamic mapping in `SumoDocument`,
|
This is because we disable dynamic mapping in `SumoDocument`,
|
||||||
to prevent a dynamic mapping of the wrong type being set up before `es7_init` was able to be run during a deployment.
|
to prevent a dynamic mapping of the wrong type being set up before `es_init` was able to be run during a deployment.
|
||||||
|
|
||||||
So to ensure no data is missing from the index,
|
So to ensure no data is missing from the index,
|
||||||
run something like:
|
run something like:
|
||||||
|
|
||||||
```
|
```
|
||||||
./manage.py es7_reindex --limit TestDocument --updated-after <datetime of deploy> --updated-before <datetime of mapping update>
|
./manage.py es_reindex --limit TestDocument --updated-after <datetime of deploy> --updated-before <datetime of mapping update>
|
||||||
```
|
```
|
||||||
|
|
||||||
### Indexing performance
|
### Indexing performance
|
||||||
|
@ -45,7 +45,7 @@ you might want to add the `--print-sql-count` argument when testing out your cha
|
||||||
to see how many SQL queries are being executed:
|
to see how many SQL queries are being executed:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
CELERY_TASK_ALWAYS_EAGER=True ./manage.py es7_reindex --print-sql-count --sql-chunk-size=100 --count=100
|
CELERY_TASK_ALWAYS_EAGER=True ./manage.py es_reindex --print-sql-count --sql-chunk-size=100 --count=100
|
||||||
```
|
```
|
||||||
|
|
||||||
If the result is much less than 100,
|
If the result is much less than 100,
|
||||||
|
@ -87,7 +87,7 @@ as that returns a naive or aware datetime depending on the value of `USE_TZ`, wh
|
||||||
You can set the following variable in your .env file to enable the logging of the queries that are sent to your local ElasticSearch instance.
|
You can set the following variable in your .env file to enable the logging of the queries that are sent to your local ElasticSearch instance.
|
||||||
|
|
||||||
```
|
```
|
||||||
ES7_ENABLE_CONSOLE_LOGGING=True
|
ES_ENABLE_CONSOLE_LOGGING=True
|
||||||
```
|
```
|
||||||
|
|
||||||
### Simulate slow and out of order query responses
|
### Simulate slow and out of order query responses
|
||||||
|
@ -234,7 +234,7 @@ In development synonyms can be updated very easily.
|
||||||
Save your changes in the text file and run:
|
Save your changes in the text file and run:
|
||||||
|
|
||||||
```
|
```
|
||||||
./manage.py es7_init --reload-search-analyzers
|
./manage.py es_init --reload-search-analyzers
|
||||||
```
|
```
|
||||||
|
|
||||||
If no other changes were made to the index configurations,
|
If no other changes were made to the index configurations,
|
||||||
|
|
|
@ -74,7 +74,7 @@ app:
|
||||||
dms_fix_current_revisions: SECRET
|
dms_fix_current_revisions: SECRET
|
||||||
dms_generate_missing_share_links: SECRET
|
dms_generate_missing_share_links: SECRET
|
||||||
dms_rebuild_kb: SECRET
|
dms_rebuild_kb: SECRET
|
||||||
dms_reindex_es7: SECRET
|
dms_reindex_es: SECRET
|
||||||
dms_reload_question_traffic_stats: SECRET
|
dms_reload_question_traffic_stats: SECRET
|
||||||
dms_reload_wiki_traffic_stats: SECRET
|
dms_reload_wiki_traffic_stats: SECRET
|
||||||
dms_report_employee_answers: SECRET
|
dms_report_employee_answers: SECRET
|
||||||
|
|
|
@ -69,7 +69,7 @@ app:
|
||||||
dms_fix_current_revisions: SECRET
|
dms_fix_current_revisions: SECRET
|
||||||
dms_generate_missing_share_links: SECRET
|
dms_generate_missing_share_links: SECRET
|
||||||
dms_rebuild_kb: SECRET
|
dms_rebuild_kb: SECRET
|
||||||
dms_reindex_es7: SECRET
|
dms_reindex_es: SECRET
|
||||||
dms_reload_question_traffic_stats: SECRET
|
dms_reload_question_traffic_stats: SECRET
|
||||||
dms_reload_wiki_traffic_stats: SECRET
|
dms_reload_wiki_traffic_stats: SECRET
|
||||||
dms_report_employee_answers: SECRET
|
dms_report_employee_answers: SECRET
|
||||||
|
@ -93,9 +93,9 @@ app:
|
||||||
engage_robots: True
|
engage_robots: True
|
||||||
es_index_prefix: "sumo_prod"
|
es_index_prefix: "sumo_prod"
|
||||||
es_live_indexing: True
|
es_live_indexing: True
|
||||||
es7_use_ssl: True
|
es_use_ssl: True
|
||||||
es7_http_auth: SECRET
|
es_http_auth: SECRET
|
||||||
es7_cloud_id: SECRET
|
es_cloud_id: SECRET
|
||||||
ga_account: SECRET
|
ga_account: SECRET
|
||||||
ga_key: SECRET
|
ga_key: SECRET
|
||||||
ga_profile_id: SECRET
|
ga_profile_id: SECRET
|
||||||
|
|
|
@ -72,7 +72,7 @@ app:
|
||||||
dms_fix_current_revisions: SECRET
|
dms_fix_current_revisions: SECRET
|
||||||
dms_generate_missing_share_links: SECRET
|
dms_generate_missing_share_links: SECRET
|
||||||
dms_rebuild_kb: SECRET
|
dms_rebuild_kb: SECRET
|
||||||
dms_reindex_es7: SECRET
|
dms_reindex_es: SECRET
|
||||||
dms_reload_question_traffic_stats: SECRET
|
dms_reload_question_traffic_stats: SECRET
|
||||||
dms_reload_wiki_traffic_stats: SECRET
|
dms_reload_wiki_traffic_stats: SECRET
|
||||||
dms_report_employee_answers: SECRET
|
dms_report_employee_answers: SECRET
|
||||||
|
@ -95,9 +95,9 @@ app:
|
||||||
engage_robots: False
|
engage_robots: False
|
||||||
es_index_prefix: "sumo_stage"
|
es_index_prefix: "sumo_stage"
|
||||||
es_live_indexing: True
|
es_live_indexing: True
|
||||||
es7_use_ssl: True
|
es_use_ssl: True
|
||||||
es7_http_auth: SECRET
|
es_http_auth: SECRET
|
||||||
es7_cloud_id: SECRET
|
es_cloud_id: SECRET
|
||||||
ga_account: SECRET
|
ga_account: SECRET
|
||||||
ga_key: SECRET
|
ga_key: SECRET
|
||||||
ga_profile_id: SECRET
|
ga_profile_id: SECRET
|
||||||
|
|
|
@ -69,7 +69,7 @@ app:
|
||||||
dms_fix_current_revisions: SECRET
|
dms_fix_current_revisions: SECRET
|
||||||
dms_generate_missing_share_links: SECRET
|
dms_generate_missing_share_links: SECRET
|
||||||
dms_rebuild_kb: SECRET
|
dms_rebuild_kb: SECRET
|
||||||
dms_reindex_es7: SECRET
|
dms_reindex_es: SECRET
|
||||||
dms_reload_question_traffic_stats: SECRET
|
dms_reload_question_traffic_stats: SECRET
|
||||||
dms_reload_wiki_traffic_stats: SECRET
|
dms_reload_wiki_traffic_stats: SECRET
|
||||||
dms_report_employee_answers: SECRET
|
dms_report_employee_answers: SECRET
|
||||||
|
|
|
@ -73,7 +73,7 @@ app:
|
||||||
dms_fix_current_revisions: SECRET
|
dms_fix_current_revisions: SECRET
|
||||||
dms_generate_missing_share_links: SECRET
|
dms_generate_missing_share_links: SECRET
|
||||||
dms_rebuild_kb: SECRET
|
dms_rebuild_kb: SECRET
|
||||||
dms_reindex_es7: SECRET
|
dms_reindex_es: SECRET
|
||||||
dms_reload_question_traffic_stats: SECRET
|
dms_reload_question_traffic_stats: SECRET
|
||||||
dms_reload_wiki_traffic_stats: SECRET
|
dms_reload_wiki_traffic_stats: SECRET
|
||||||
dms_report_employee_answers: SECRET
|
dms_report_employee_answers: SECRET
|
||||||
|
@ -99,9 +99,9 @@ app:
|
||||||
engage_robots: True
|
engage_robots: True
|
||||||
es_index_prefix: "sumo_prod"
|
es_index_prefix: "sumo_prod"
|
||||||
es_live_indexing: True
|
es_live_indexing: True
|
||||||
es7_use_ssl: True
|
es_use_ssl: True
|
||||||
es7_http_auth: SECRET
|
es_http_auth: SECRET
|
||||||
es7_cloud_id: SECRET
|
es_cloud_id: SECRET
|
||||||
ga_account: SECRET
|
ga_account: SECRET
|
||||||
ga_key: SECRET
|
ga_key: SECRET
|
||||||
ga_profile_id: SECRET
|
ga_profile_id: SECRET
|
||||||
|
|
|
@ -72,7 +72,7 @@ app:
|
||||||
dms_fix_current_revisions: SECRET
|
dms_fix_current_revisions: SECRET
|
||||||
dms_generate_missing_share_links: SECRET
|
dms_generate_missing_share_links: SECRET
|
||||||
dms_rebuild_kb: SECRET
|
dms_rebuild_kb: SECRET
|
||||||
dms_reindex_es7: SECRET
|
dms_reindex_es: SECRET
|
||||||
dms_reload_question_traffic_stats: SECRET
|
dms_reload_question_traffic_stats: SECRET
|
||||||
dms_reload_wiki_traffic_stats: SECRET
|
dms_reload_wiki_traffic_stats: SECRET
|
||||||
dms_report_employee_answers: SECRET
|
dms_report_employee_answers: SECRET
|
||||||
|
@ -95,9 +95,9 @@ app:
|
||||||
engage_robots: False
|
engage_robots: False
|
||||||
es_index_prefix: "sumo_stage"
|
es_index_prefix: "sumo_stage"
|
||||||
es_live_indexing: True
|
es_live_indexing: True
|
||||||
es7_use_ssl: True
|
es_use_ssl: True
|
||||||
es7_http_auth: SECRET
|
es_http_auth: SECRET
|
||||||
es7_cloud_id: SECRET
|
es_cloud_id: SECRET
|
||||||
fxa_op_token_endpoint: SECRET
|
fxa_op_token_endpoint: SECRET
|
||||||
fxa_set_issuer: SECRET
|
fxa_set_issuer: SECRET
|
||||||
fxa_op_authorization_endpoint: SECRET
|
fxa_op_authorization_endpoint: SECRET
|
||||||
|
|
|
@ -6,7 +6,7 @@ from django.core.management.base import BaseCommand
|
||||||
from django.db import connection, transaction
|
from django.db import connection, transaction
|
||||||
|
|
||||||
from kitsune.questions.models import Answer, Question
|
from kitsune.questions.models import Answer, Question
|
||||||
from kitsune.search.es7_utils import index_objects_bulk
|
from kitsune.search.es_utils import index_objects_bulk
|
||||||
|
|
||||||
log = logging.getLogger("k.cron")
|
log = logging.getLogger("k.cron")
|
||||||
|
|
||||||
|
|
|
@ -18,11 +18,11 @@ from elasticsearch_dsl.utils import AttrDict
|
||||||
from pyparsing import ParseException
|
from pyparsing import ParseException
|
||||||
|
|
||||||
from kitsune.search.config import (
|
from kitsune.search.config import (
|
||||||
DEFAULT_ES7_CONNECTION,
|
DEFAULT_ES_CONNECTION,
|
||||||
DEFAULT_ES7_REFRESH_INTERVAL,
|
DEFAULT_ES_REFRESH_INTERVAL,
|
||||||
UPDATE_RETRY_ON_CONFLICT,
|
UPDATE_RETRY_ON_CONFLICT,
|
||||||
)
|
)
|
||||||
from kitsune.search.es7_utils import es7_client
|
from kitsune.search.es_utils import es_client
|
||||||
from kitsune.search.parser import Parser
|
from kitsune.search.parser import Parser
|
||||||
from kitsune.search.parser.tokens import TermToken
|
from kitsune.search.parser.tokens import TermToken
|
||||||
|
|
||||||
|
@ -40,7 +40,7 @@ class SumoDocument(DSLDocument):
|
||||||
"""Automatically set up each subclass' Index attribute."""
|
"""Automatically set up each subclass' Index attribute."""
|
||||||
super().__init_subclass__(**kwargs)
|
super().__init_subclass__(**kwargs)
|
||||||
|
|
||||||
cls.Index.using = DEFAULT_ES7_CONNECTION
|
cls.Index.using = DEFAULT_ES_CONNECTION
|
||||||
|
|
||||||
# this is here to ensure subclasses of subclasses of SumoDocument (e.g. AnswerDocument)
|
# this is here to ensure subclasses of subclasses of SumoDocument (e.g. AnswerDocument)
|
||||||
# use the same name in their index as their parent class (e.g. QuestionDocument) since
|
# use the same name in their index as their parent class (e.g. QuestionDocument) since
|
||||||
|
@ -55,7 +55,7 @@ class SumoDocument(DSLDocument):
|
||||||
cls.Index.read_alias = f"{cls.Index.base_name}_read"
|
cls.Index.read_alias = f"{cls.Index.base_name}_read"
|
||||||
cls.Index.write_alias = f"{cls.Index.base_name}_write"
|
cls.Index.write_alias = f"{cls.Index.base_name}_write"
|
||||||
# Bump the refresh interval to 1 minute
|
# Bump the refresh interval to 1 minute
|
||||||
cls.Index.settings = {"refresh_interval": DEFAULT_ES7_REFRESH_INTERVAL}
|
cls.Index.settings = {"refresh_interval": DEFAULT_ES_REFRESH_INTERVAL}
|
||||||
|
|
||||||
# this is the attribute elastic-dsl actually uses to determine which index
|
# this is the attribute elastic-dsl actually uses to determine which index
|
||||||
# to query. we override the .search() method to get that to use the read
|
# to query. we override the .search() method to get that to use the read
|
||||||
|
@ -88,7 +88,7 @@ class SumoDocument(DSLDocument):
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def _update_alias(cls, alias, new_index):
|
def _update_alias(cls, alias, new_index):
|
||||||
client = es7_client()
|
client = es_client()
|
||||||
old_index = cls.alias_points_at(alias)
|
old_index = cls.alias_points_at(alias)
|
||||||
if not old_index:
|
if not old_index:
|
||||||
client.indices.put_alias(new_index, alias)
|
client.indices.put_alias(new_index, alias)
|
||||||
|
@ -106,7 +106,7 @@ class SumoDocument(DSLDocument):
|
||||||
def alias_points_at(cls, alias):
|
def alias_points_at(cls, alias):
|
||||||
"""Returns the index `alias` points at."""
|
"""Returns the index `alias` points at."""
|
||||||
try:
|
try:
|
||||||
aliased_indices = list(es7_client().indices.get_alias(name=alias))
|
aliased_indices = list(es_client().indices.get_alias(name=alias))
|
||||||
except NotFoundError:
|
except NotFoundError:
|
||||||
aliased_indices = []
|
aliased_indices = []
|
||||||
|
|
||||||
|
@ -376,8 +376,8 @@ class SumoSearch(SumoSearchInterface):
|
||||||
"""Perform search, placing the results in `self.results`, and the total
|
"""Perform search, placing the results in `self.results`, and the total
|
||||||
number of results (across all pages) in `self.total`. Chainable."""
|
number of results (across all pages) in `self.total`. Chainable."""
|
||||||
|
|
||||||
search = DSLSearch(using=es7_client(), index=self.get_index()).params(
|
search = DSLSearch(using=es_client(), index=self.get_index()).params(
|
||||||
**settings.ES7_SEARCH_PARAMS
|
**settings.ES_SEARCH_PARAMS
|
||||||
)
|
)
|
||||||
|
|
||||||
# add the search class' filter
|
# add the search class' filter
|
||||||
|
|
|
@ -371,6 +371,6 @@ ES_LOCALE_ANALYZERS = {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
DEFAULT_ES7_CONNECTION = "es7_default"
|
DEFAULT_ES_CONNECTION = "es_default"
|
||||||
# default refresh_interval for all indices
|
# default refresh_interval for all indices
|
||||||
DEFAULT_ES7_REFRESH_INTERVAL = "60s"
|
DEFAULT_ES_REFRESH_INTERVAL = "60s"
|
||||||
|
|
|
@ -5,7 +5,7 @@ from kitsune.forums.models import Post
|
||||||
from kitsune.questions.models import Answer, Question
|
from kitsune.questions.models import Answer, Question
|
||||||
from kitsune.search import config
|
from kitsune.search import config
|
||||||
from kitsune.search.base import SumoDocument
|
from kitsune.search.base import SumoDocument
|
||||||
from kitsune.search.es7_utils import es7_client
|
from kitsune.search.es_utils import es_client
|
||||||
from kitsune.search.fields import SumoLocaleAwareKeywordField, SumoLocaleAwareTextField
|
from kitsune.search.fields import SumoLocaleAwareKeywordField, SumoLocaleAwareTextField
|
||||||
from kitsune.users.models import Profile
|
from kitsune.users.models import Profile
|
||||||
from kitsune.wiki import models as wiki_models
|
from kitsune.wiki import models as wiki_models
|
||||||
|
@ -16,7 +16,7 @@ from kitsune.wiki.config import (
|
||||||
TEMPLATES_CATEGORY,
|
TEMPLATES_CATEGORY,
|
||||||
)
|
)
|
||||||
|
|
||||||
connections.add_connection(config.DEFAULT_ES7_CONNECTION, es7_client())
|
connections.add_connection(config.DEFAULT_ES_CONNECTION, es_client())
|
||||||
|
|
||||||
|
|
||||||
class WikiDocument(SumoDocument):
|
class WikiDocument(SumoDocument):
|
||||||
|
|
|
@ -4,7 +4,7 @@ import inspect
|
||||||
from celery import shared_task
|
from celery import shared_task
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from elasticsearch import Elasticsearch
|
from elasticsearch import Elasticsearch
|
||||||
from elasticsearch.helpers import bulk as es7_bulk
|
from elasticsearch.helpers import bulk as es_bulk
|
||||||
from elasticsearch.helpers.errors import BulkIndexError
|
from elasticsearch.helpers.errors import BulkIndexError
|
||||||
from elasticsearch_dsl import Document, UpdateByQuery, analyzer, char_filter, token_filter
|
from elasticsearch_dsl import Document, UpdateByQuery, analyzer, char_filter, token_filter
|
||||||
|
|
||||||
|
@ -90,13 +90,13 @@ def es_analyzer_for_locale(locale, search_analyzer=False):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def es7_client(**kwargs):
|
def es_client(**kwargs):
|
||||||
"""Return an ES7 Elasticsearch client"""
|
"""Return an ES Elasticsearch client"""
|
||||||
# prefer a cloud_id if available
|
# prefer a cloud_id if available
|
||||||
if es7_cloud_id := settings.ES7_CLOUD_ID:
|
if es_cloud_id := settings.ES_CLOUD_ID:
|
||||||
kwargs.update({"cloud_id": es7_cloud_id, "http_auth": settings.ES7_HTTP_AUTH})
|
kwargs.update({"cloud_id": es_cloud_id, "http_auth": settings.ES_HTTP_AUTH})
|
||||||
else:
|
else:
|
||||||
kwargs.update({"hosts": settings.ES7_URLS})
|
kwargs.update({"hosts": settings.ES_URLS})
|
||||||
return Elasticsearch(**kwargs)
|
return Elasticsearch(**kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
@ -166,8 +166,8 @@ def index_objects_bulk(
|
||||||
# if the request doesn't resolve within `timeout`,
|
# if the request doesn't resolve within `timeout`,
|
||||||
# sleep for `timeout` then try again up to `settings.ES_BULK_MAX_RETRIES` times,
|
# sleep for `timeout` then try again up to `settings.ES_BULK_MAX_RETRIES` times,
|
||||||
# before raising an exception:
|
# before raising an exception:
|
||||||
success, errors = es7_bulk(
|
success, errors = es_bulk(
|
||||||
es7_client(
|
es_client(
|
||||||
timeout=timeout,
|
timeout=timeout,
|
||||||
retry_on_timeout=True,
|
retry_on_timeout=True,
|
||||||
initial_backoff=timeout,
|
initial_backoff=timeout,
|
||||||
|
@ -198,7 +198,7 @@ def remove_from_field(doc_type_name, field_name, field_value):
|
||||||
f"}}"
|
f"}}"
|
||||||
)
|
)
|
||||||
|
|
||||||
update = UpdateByQuery(using=es7_client(), index=doc_type._index._name)
|
update = UpdateByQuery(using=es_client(), index=doc_type._index._name)
|
||||||
update = update.filter("term", **{field_name: field_value})
|
update = update.filter("term", **{field_name: field_value})
|
||||||
update = update.script(source=script, params={"value": field_value}, conflicts="proceed")
|
update = update.script(source=script, params={"value": field_value}, conflicts="proceed")
|
||||||
|
|
|
@ -5,7 +5,7 @@ from elasticsearch_dsl.field import Keyword
|
||||||
from elasticsearch_dsl.field import Object as DSLObject
|
from elasticsearch_dsl.field import Object as DSLObject
|
||||||
from elasticsearch_dsl.field import Text
|
from elasticsearch_dsl.field import Text
|
||||||
|
|
||||||
from kitsune.search.es7_utils import es_analyzer_for_locale
|
from kitsune.search.es_utils import es_analyzer_for_locale
|
||||||
|
|
||||||
SUPPORTED_LANGUAGES = list(settings.SUMO_LANGUAGES)
|
SUPPORTED_LANGUAGES = list(settings.SUMO_LANGUAGES)
|
||||||
# this is a test locale - no need to add it to ES
|
# this is a test locale - no need to add it to ES
|
||||||
|
|
|
@ -2,11 +2,11 @@ from django.core.management.base import BaseCommand
|
||||||
from elasticsearch_dsl.exceptions import IllegalOperation
|
from elasticsearch_dsl.exceptions import IllegalOperation
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
from kitsune.search.es7_utils import get_doc_types, es7_client
|
from kitsune.search.es_utils import get_doc_types, es_client
|
||||||
|
|
||||||
|
|
||||||
class Command(BaseCommand):
|
class Command(BaseCommand):
|
||||||
help = "Initialize ES7 document types"
|
help = "Initialize ES document types"
|
||||||
|
|
||||||
def add_arguments(self, parser):
|
def add_arguments(self, parser):
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
|
@ -34,7 +34,7 @@ class Command(BaseCommand):
|
||||||
)
|
)
|
||||||
|
|
||||||
def handle(self, *args, **kwargs):
|
def handle(self, *args, **kwargs):
|
||||||
client = es7_client()
|
client = es_client()
|
||||||
doc_types = get_doc_types()
|
doc_types = get_doc_types()
|
||||||
|
|
||||||
limit = kwargs["limit"]
|
limit = kwargs["limit"]
|
|
@ -5,11 +5,11 @@ from django.core.management.base import BaseCommand
|
||||||
from django.db import connection, reset_queries
|
from django.db import connection, reset_queries
|
||||||
from dateutil.parser import parse as dateutil_parse
|
from dateutil.parser import parse as dateutil_parse
|
||||||
|
|
||||||
from kitsune.search.es7_utils import get_doc_types, index_objects_bulk
|
from kitsune.search.es_utils import get_doc_types, index_objects_bulk
|
||||||
|
|
||||||
|
|
||||||
class Command(BaseCommand):
|
class Command(BaseCommand):
|
||||||
help = "Reindex ES7 documents"
|
help = "Reindex ES documents"
|
||||||
|
|
||||||
def add_arguments(self, parser):
|
def add_arguments(self, parser):
|
||||||
parser.add_argument(
|
parser.add_argument(
|
|
@ -1,5 +1,5 @@
|
||||||
from django.db.models.signals import post_save, post_delete
|
from django.db.models.signals import post_save, post_delete
|
||||||
from kitsune.search.es7_utils import index_object, delete_object, index_objects_bulk
|
from kitsune.search.es_utils import index_object, delete_object, index_objects_bulk
|
||||||
from kitsune.search.decorators import search_receiver
|
from kitsune.search.decorators import search_receiver
|
||||||
from kitsune.forums.models import Thread, Post
|
from kitsune.forums.models import Thread, Post
|
||||||
|
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
from django.db.models.signals import post_save, post_delete, m2m_changed
|
from django.db.models.signals import post_save, post_delete, m2m_changed
|
||||||
from kitsune.search.es7_utils import (
|
from kitsune.search.es_utils import (
|
||||||
index_object,
|
index_object,
|
||||||
delete_object,
|
delete_object,
|
||||||
index_objects_bulk,
|
index_objects_bulk,
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
from django.db.models.signals import post_save, post_delete, m2m_changed
|
from django.db.models.signals import post_save, post_delete, m2m_changed
|
||||||
from django.contrib.auth.models import User, Group
|
from django.contrib.auth.models import User, Group
|
||||||
from kitsune.users.models import Profile
|
from kitsune.users.models import Profile
|
||||||
from kitsune.search.es7_utils import (
|
from kitsune.search.es_utils import (
|
||||||
index_object,
|
index_object,
|
||||||
delete_object,
|
delete_object,
|
||||||
remove_from_field,
|
remove_from_field,
|
||||||
|
|
|
@ -2,7 +2,7 @@ from django.db.models.signals import m2m_changed, post_delete, post_save
|
||||||
|
|
||||||
from kitsune.products.models import Product, Topic
|
from kitsune.products.models import Product, Topic
|
||||||
from kitsune.search.decorators import search_receiver
|
from kitsune.search.decorators import search_receiver
|
||||||
from kitsune.search.es7_utils import delete_object, index_object, remove_from_field
|
from kitsune.search.es_utils import delete_object, index_object, remove_from_field
|
||||||
from kitsune.wiki.models import Document
|
from kitsune.wiki.models import Document
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
from django.test.utils import override_settings
|
from django.test.utils import override_settings
|
||||||
|
|
||||||
from kitsune.search.es7_utils import get_doc_types
|
from kitsune.search.es_utils import get_doc_types
|
||||||
from kitsune.sumo.tests import TestCase
|
from kitsune.sumo.tests import TestCase
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -2,8 +2,8 @@ from django.test.utils import override_settings
|
||||||
from kitsune.search.tests import Elastic7TestCase
|
from kitsune.search.tests import Elastic7TestCase
|
||||||
from kitsune.search.documents import ProfileDocument
|
from kitsune.search.documents import ProfileDocument
|
||||||
from kitsune.users.tests import ProfileFactory, GroupFactory
|
from kitsune.users.tests import ProfileFactory, GroupFactory
|
||||||
from elasticsearch.helpers import bulk as es7_bulk
|
from elasticsearch.helpers import bulk as es_bulk
|
||||||
from kitsune.search.es7_utils import es7_client
|
from kitsune.search.es_utils import es_client
|
||||||
|
|
||||||
|
|
||||||
@override_settings(ES_LIVE_INDEXING=False)
|
@override_settings(ES_LIVE_INDEXING=False)
|
||||||
|
@ -28,7 +28,7 @@ class ToActionTests(Elastic7TestCase):
|
||||||
|
|
||||||
def test_index_bulk_empty_list(self):
|
def test_index_bulk_empty_list(self):
|
||||||
payload = self.prepare().to_action("index", is_bulk=True)
|
payload = self.prepare().to_action("index", is_bulk=True)
|
||||||
es7_bulk(es7_client(), [payload])
|
es_bulk(es_client(), [payload])
|
||||||
self.assertEqual(self.doc.group_ids, [])
|
self.assertEqual(self.doc.group_ids, [])
|
||||||
|
|
||||||
def test_update_empty_list(self):
|
def test_update_empty_list(self):
|
||||||
|
@ -37,5 +37,5 @@ class ToActionTests(Elastic7TestCase):
|
||||||
|
|
||||||
def test_update_bulk_empty_list(self):
|
def test_update_bulk_empty_list(self):
|
||||||
payload = self.prepare().to_action("update", is_bulk=True)
|
payload = self.prepare().to_action("update", is_bulk=True)
|
||||||
es7_bulk(es7_client(), [payload])
|
es_bulk(es_client(), [payload])
|
||||||
self.assertEqual(self.doc.group_ids, [])
|
self.assertEqual(self.doc.group_ids, [])
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
from kitsune.search.tests import Elastic7TestCase
|
from kitsune.search.tests import Elastic7TestCase
|
||||||
from kitsune.questions.tests import QuestionFactory
|
from kitsune.questions.tests import QuestionFactory
|
||||||
from django.test.utils import override_settings
|
from django.test.utils import override_settings
|
||||||
from kitsune.search.es7_utils import index_objects_bulk
|
from kitsune.search.es_utils import index_objects_bulk
|
||||||
from elasticsearch.helpers.errors import BulkIndexError
|
from elasticsearch.helpers.errors import BulkIndexError
|
||||||
from elasticsearch.exceptions import NotFoundError
|
from elasticsearch.exceptions import NotFoundError
|
||||||
from kitsune.search.documents import QuestionDocument
|
from kitsune.search.documents import QuestionDocument
|
||||||
|
|
|
@ -738,14 +738,14 @@ CSRF_COOKIE_SECURE = config("CSRF_COOKIE_SECURE", default=not DEBUG, cast=bool)
|
||||||
#
|
#
|
||||||
# Connection information for Elastic 7
|
# Connection information for Elastic 7
|
||||||
ES_TIMEOUT = 5 # Timeout for querying requests
|
ES_TIMEOUT = 5 # Timeout for querying requests
|
||||||
ES7_URLS = config("ES7_URLS", cast=Csv(), default="elasticsearch7:9200")
|
ES_URLS = config("ES_URLS", cast=Csv(), default="elasticsearch:9200")
|
||||||
ES7_CLOUD_ID = config("ES7_CLOUD_ID", default="")
|
ES_CLOUD_ID = config("ES_CLOUD_ID", default="")
|
||||||
ES7_USE_SSL = config("ES7_USE_SSL", default=False, cast=bool)
|
ES_USE_SSL = config("ES_USE_SSL", default=False, cast=bool)
|
||||||
ES7_HTTP_AUTH = config("ES7_HTTP_AUTH", default="", cast=Csv())
|
ES_HTTP_AUTH = config("ES_HTTP_AUTH", default="", cast=Csv())
|
||||||
ES7_ENABLE_CONSOLE_LOGGING = config("ES7_ENABLE_CONSOLE_LOGGING", default=False, cast=bool)
|
ES_ENABLE_CONSOLE_LOGGING = config("ES_ENABLE_CONSOLE_LOGGING", default=False, cast=bool)
|
||||||
# Pass parameters to the ES7 client
|
# Pass parameters to the ES client
|
||||||
# like "search_type": "dfs_query_then_fetch"
|
# like "search_type": "dfs_query_then_fetch"
|
||||||
ES7_SEARCH_PARAMS = {"request_timeout": ES_TIMEOUT}
|
ES_SEARCH_PARAMS = {"request_timeout": ES_TIMEOUT}
|
||||||
|
|
||||||
# This is prepended to index names to get the final read/write index
|
# This is prepended to index names to get the final read/write index
|
||||||
# names used by kitsune. This is so that you can have multiple
|
# names used by kitsune. This is so that you can have multiple
|
||||||
|
@ -1059,7 +1059,7 @@ DMS_UPDATE_SEARCH_CTR_METRIC = config("DMS_UPDATE_SEARCH_CTR_METRIC", default=No
|
||||||
DMS_UPDATE_CONTRIBUTOR_METRICS = config("DMS_UPDATE_CONTRIBUTOR_METRICS", default=None)
|
DMS_UPDATE_CONTRIBUTOR_METRICS = config("DMS_UPDATE_CONTRIBUTOR_METRICS", default=None)
|
||||||
DMS_AUTO_ARCHIVE_OLD_QUESTIONS = config("DMS_AUTO_ARCHIVE_OLD_QUESTIONS", default=None)
|
DMS_AUTO_ARCHIVE_OLD_QUESTIONS = config("DMS_AUTO_ARCHIVE_OLD_QUESTIONS", default=None)
|
||||||
DMS_REINDEX = config("DMS_REINDEX", default=None)
|
DMS_REINDEX = config("DMS_REINDEX", default=None)
|
||||||
DMS_REINDEX_ES7 = config("DMS_REINDEX_ES7", default=None)
|
DMS_REINDEX_ES = config("DMS_REINDEX_ES", default=None)
|
||||||
# DMS_PROCESS_EXIT_SURVEYS = config("DMS_PROCESS_EXIT_SURVEYS", default=None)
|
# DMS_PROCESS_EXIT_SURVEYS = config("DMS_PROCESS_EXIT_SURVEYS", default=None)
|
||||||
# DMS_SURVEY_RECENT_ASKERS = config("DMS_SURVEY_RECENT_ASKERS", default=None)
|
# DMS_SURVEY_RECENT_ASKERS = config("DMS_SURVEY_RECENT_ASKERS", default=None)
|
||||||
# DMS_UPDATE_VISITORS_METRIC = config('DMS_UPDATE_VISITORS_METRIC', default=None)
|
# DMS_UPDATE_VISITORS_METRIC = config('DMS_UPDATE_VISITORS_METRIC', default=None)
|
||||||
|
@ -1123,7 +1123,7 @@ TOLL_FREE_REGEX = re.compile(r"^.*8(00|33|44|55|66|77|88)[2-9]\d{6,}$")
|
||||||
REGEX_TIMEOUT = config("REGEX_TIMEOUT", default=5, cast=int)
|
REGEX_TIMEOUT = config("REGEX_TIMEOUT", default=5, cast=int)
|
||||||
NANP_REGEX = re.compile(r"[0-9]{3}-?[a-zA-Z2-9][a-zA-Z0-9]{2}-?[a-zA-Z0-9]{4}")
|
NANP_REGEX = re.compile(r"[0-9]{3}-?[a-zA-Z2-9][a-zA-Z0-9]{2}-?[a-zA-Z0-9]{4}")
|
||||||
|
|
||||||
if ES7_ENABLE_CONSOLE_LOGGING and DEV:
|
if ES_ENABLE_CONSOLE_LOGGING and DEV:
|
||||||
es_trace_logger = logging.getLogger("elasticsearch.trace")
|
es_trace_logger = logging.getLogger("elasticsearch.trace")
|
||||||
es_trace_logger.setLevel(logging.INFO)
|
es_trace_logger.setLevel(logging.INFO)
|
||||||
handler = logging.StreamHandler()
|
handler = logging.StreamHandler()
|
||||||
|
|
|
@ -85,11 +85,11 @@ def job_send_welcome_emails():
|
||||||
# only run on readonly clusters, where no signals will be triggered:
|
# only run on readonly clusters, where no signals will be triggered:
|
||||||
skip=(not settings.READ_ONLY),
|
skip=(not settings.READ_ONLY),
|
||||||
)
|
)
|
||||||
@babis.decorator(ping_after=settings.DMS_REINDEX_ES7)
|
@babis.decorator(ping_after=settings.DMS_REINDEX_ES)
|
||||||
def job_reindex_es7():
|
def job_reindex_es():
|
||||||
# Index items newer than 90 minutes old in ES7
|
# Index items newer than 90 minutes old in ES
|
||||||
after = (timezone.now() - datetime.timedelta(minutes=90)).isoformat()
|
after = (timezone.now() - datetime.timedelta(minutes=90)).isoformat()
|
||||||
call_command("es7_reindex --updated-after {}".format(after))
|
call_command("es_reindex --updated-after {}".format(after))
|
||||||
|
|
||||||
|
|
||||||
# Every 6 hours.
|
# Every 6 hours.
|
||||||
|
|
Загрузка…
Ссылка в новой задаче