mozilla
/
kitsune
зеркало из https://github.com/mozilla/kitsune.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Remove ES2 from users app.

This commit is contained in:
Tasos Katsoulas 2021-04-28 15:34:40 +03:00
Родитель 82b1745c00
Коммит 9ddb35c20f
15 изменённых файлов: 1 добавлений и 365 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

1
k8s/regions/frankfurt/dev.yaml
Просмотреть файл

@ -71,7 +71,6 @@ app:
dms_rebuild_kb: SECRET
dms_reindex: SECRET
dms_reindex_es7: SECRET
dms_reindex_users_that_contributed_yesterday: SECRET
dms_reload_question_traffic_stats: SECRET
dms_reload_wiki_traffic_stats: SECRET
dms_report_employee_answers: SECRET

1
k8s/regions/frankfurt/prod.yaml
Просмотреть файл

@ -69,7 +69,6 @@ app:
dms_rebuild_kb: SECRET
dms_reindex: SECRET
dms_reindex_es7: SECRET
dms_reindex_users_that_contributed_yesterday: SECRET
dms_reload_question_traffic_stats: SECRET
dms_reload_wiki_traffic_stats: SECRET
dms_report_employee_answers: SECRET

1
k8s/regions/frankfurt/stage.yaml
Просмотреть файл

@ -72,7 +72,6 @@ app:
dms_rebuild_kb: SECRET
dms_reindex: SECRET
dms_reindex_es7: SECRET
dms_reindex_users_that_contributed_yesterday: SECRET
dms_reload_question_traffic_stats: SECRET
dms_reload_wiki_traffic_stats: SECRET
dms_report_employee_answers: SECRET

1
k8s/regions/frankfurt/test.yaml
Просмотреть файл

@ -71,7 +71,6 @@ app:
dms_rebuild_kb: SECRET
dms_reindex: SECRET
dms_reindex_es7: SECRET
dms_reindex_users_that_contributed_yesterday: SECRET
dms_reload_question_traffic_stats: SECRET
dms_reload_wiki_traffic_stats: SECRET
dms_report_employee_answers: SECRET

1
k8s/regions/oregon-a/prod.yaml
Просмотреть файл

@ -73,7 +73,6 @@ app:
dms_rebuild_kb: SECRET
dms_reindex: SECRET
dms_reindex_es7: SECRET
dms_reindex_users_that_contributed_yesterday: SECRET
dms_reload_question_traffic_stats: SECRET
dms_reload_wiki_traffic_stats: SECRET
dms_report_employee_answers: SECRET

1
k8s/regions/oregon-a/stage.yaml
Просмотреть файл

@ -72,7 +72,6 @@ app:
dms_rebuild_kb: SECRET
dms_reindex: SECRET
dms_reindex_es7: SECRET
dms_reindex_users_that_contributed_yesterday: SECRET
dms_reload_question_traffic_stats: SECRET
dms_reload_wiki_traffic_stats: SECRET
dms_report_employee_answers: SECRET

1
k8s/regions/oregon-b/prod.yaml
Просмотреть файл

@ -73,7 +73,6 @@ app:
dms_rebuild_kb: SECRET
dms_reindex: SECRET
dms_reindex_es7: SECRET
dms_reindex_users_that_contributed_yesterday: SECRET
dms_reload_question_traffic_stats: SECRET
dms_reload_wiki_traffic_stats: SECRET
dms_report_employee_answers: SECRET

1
k8s/regions/oregon-b/stage.yaml
Просмотреть файл

@ -72,7 +72,6 @@ app:
dms_rebuild_kb: SECRET
dms_reindex: SECRET
dms_reindex_es7: SECRET
dms_reindex_users_that_contributed_yesterday: SECRET
dms_reload_question_traffic_stats: SECRET
dms_reload_wiki_traffic_stats: SECRET
dms_report_employee_answers: SECRET

3
kitsune/settings.py
Просмотреть файл

@ -1141,9 +1141,6 @@ DMS_UPDATE_TOP_CONTRIBUTORS = config("DMS_UPDATE_TOP_CONTRIBUTORS", default=None
DMS_UPDATE_L10N_COVERAGE_METRICS = config("DMS_UPDATE_L10N_COVERAGE_METRICS", default=None)
DMS_CALCULATE_CSAT_METRICS = config("DMS_CALCULATE_CSAT_METRICS", default=None)
DMS_REPORT_EMPLOYEE_ANSWERS = config("DMS_REPORT_EMPLOYEE_ANSWERS", default=None)
DMS_REINDEX_USERS_THAT_CONTRIBUTED_YESTERDAY = config(
"DMS_REINDEX_USERS_THAT_CONTRIBUTED_YESTERDAY", default=None
)
DMS_UPDATE_WEEKLY_VOTES = config("DMS_UPDATE_WEEKLY_VOTES", default=None)
DMS_UPDATE_SEARCH_CTR_METRIC = config("DMS_UPDATE_SEARCH_CTR_METRIC", default=None)
DMS_UPDATE_CONTRIBUTOR_METRICS = config("DMS_UPDATE_CONTRIBUTOR_METRICS", default=None)

0
kitsune/users/management/__init__.py
Просмотреть файл

0
kitsune/users/management/commands/__init__.py
Просмотреть файл

43
kitsune/users/management/commands/reindex_users_that_contributed_yesterday.py
Просмотреть файл

@ -1,43 +0,0 @@
from datetime import datetime, timedelta
from django.core.management.base import BaseCommand
from kitsune.questions.models import Answer
from kitsune.search.tasks import index_task
from kitsune.search.utils import to_class_path
from kitsune.users.models import UserMappingType
from kitsune.wiki.models import Revision
class Command(BaseCommand):
help = "Update the users (in ES) that contributed yesterday."
def handle(self, **options):
"""
The idea is to update the last_contribution_date field.
"""
today = datetime.now()
yesterday = today - timedelta(days=1)
# Support Forum answers
user_ids = list(
Answer.objects.filter(created__gte=yesterday, created__lt=today).values_list(
"creator_id", flat=True
)
)
# KB Edits
user_ids += list(
Revision.objects.filter(created__gte=yesterday, created__lt=today).values_list(
"creator_id", flat=True
)
)
# KB Reviews
user_ids += list(
Revision.objects.filter(reviewed__gte=yesterday, reviewed__lt=today).values_list(
"reviewer_id", flat=True
)
)
index_task.delay(to_class_path(UserMappingType), list(set(user_ids)))

121
kitsune/users/models.py
Просмотреть файл

@ -1,24 +1,15 @@
import logging
import re
import time
from datetime import datetime
from django.conf import settings
from django.contrib.auth.models import User
from django.db import models
from django.utils.translation import ugettext as _
from django.utils.translation import ugettext_lazy as _lazy
from timezone_field import TimeZoneField
from kitsune.lib.countries import COUNTRIES
from kitsune.products.models import Product
from kitsune.search.es_utils import UnindexMeBro
from kitsune.search.models import (
SearchMappingType,
SearchMixin,
register_for_indexing,
register_mapping_type,
)
from kitsune.sumo.models import LocaleField, ModelBase
from kitsune.sumo.urlresolvers import reverse
from kitsune.sumo.utils import auto_delete_files
@ -33,7 +24,7 @@ SET_ID_PREFIX = "https://schemas.accounts.firefox.com/event/"
@auto_delete_files
class Profile(ModelBase, SearchMixin):
class Profile(ModelBase):
"""Profile model for django users."""
user = models.OneToOneField(
@ -147,10 +138,6 @@ class Profile(ModelBase, SearchMixin):
def display_name(self):
return self.name if self.name else self.user.username
@classmethod
def get_mapping_type(cls):
return UserMappingType
@classmethod
def get_serializer(cls, serializer_type="full"):
# Avoid circular import
@ -210,112 +197,6 @@ class Profile(ModelBase, SearchMixin):
return AnswerVote.objects.filter(answer__creator=self.user, helpful=True).count()
@register_mapping_type
class UserMappingType(SearchMappingType):
list_keys = [
"twitter_usernames",
"itwitter_usernames",
]
@classmethod
def get_model(cls):
return Profile
@classmethod
def get_index_group(cls):
return "non-critical"
@classmethod
def get_mapping(cls):
return {
"properties": {
"id": {"type": "long"},
"model": {"type": "string", "index": "not_analyzed"},
"url": {"type": "string", "index": "not_analyzed"},
"indexed_on": {"type": "integer"},
"username": {"type": "string", "index": "not_analyzed"},
"display_name": {"type": "string", "index": "not_analyzed"},
"twitter_usernames": {"type": "string", "index": "not_analyzed"},
"last_contribution_date": {"type": "date"},
# lower-cased versions for querying:
"iusername": {"type": "string", "index": "not_analyzed"},
"idisplay_name": {"type": "string", "analyzer": "whitespace"},
"itwitter_usernames": {"type": "string", "index": "not_analyzed"},
"avatar": {"type": "string", "index": "not_analyzed"},
"suggest": {"type": "completion", "analyzer": "whitespace", "payloads": True},
}
}
@classmethod
def extract_document(cls, obj_id, obj=None):
"""Extracts interesting thing from a Thread and its Posts"""
if obj is None:
model = cls.get_model()
obj = model.objects.select_related("user").get(pk=obj_id)
if not obj.user.is_active:
raise UnindexMeBro()
d = {}
d["id"] = obj.pk
d["model"] = cls.get_mapping_type_name()
d["url"] = obj.get_absolute_url()
d["indexed_on"] = int(time.time())
d["username"] = obj.user.username
d["display_name"] = obj.display_name
d["twitter_usernames"] = []
d["last_contribution_date"] = obj.last_contribution_date
d["iusername"] = obj.user.username.lower()
d["idisplay_name"] = obj.display_name.lower()
d["itwitter_usernames"] = []
from kitsune.users.templatetags.jinja_helpers import profile_avatar
d["avatar"] = profile_avatar(obj.user, size=120)
d["suggest"] = {
"input": [d["iusername"], d["idisplay_name"]],
"output": _("{displayname} ({username})").format(
displayname=d["display_name"], username=d["username"]
),
"payload": {"user_id": d["id"]},
}
return d
@classmethod
def suggest_completions(cls, text):
"""Suggest completions for the text provided."""
USER_SUGGEST = "user-suggest"
es = UserMappingType.search().get_es()
results = es.suggest(
index=cls.get_index(),
body={USER_SUGGEST: {"text": text.lower(), "completion": {"field": "suggest"}}},
)
if results[USER_SUGGEST][0]["length"] > 0:
return results[USER_SUGGEST][0]["options"]
return []
register_for_indexing("users", Profile)
def get_profile(u):
try:
return Profile.objects.get(user=u)
except Profile.DoesNotExist:
return None
register_for_indexing("users", User, instance_to_indexee=get_profile)
class Setting(ModelBase):
"""User specific value per setting"""

176
kitsune/users/tests/test_es.py
Просмотреть файл

@ -1,176 +0,0 @@
# -*- coding: utf-8 -*-
from datetime import datetime, timedelta
from django.core.management import call_command
from nose.tools import eq_
from kitsune.questions.tests import AnswerFactory
from kitsune.search.tests.test_es import ElasticTestCase
from kitsune.users.models import UserMappingType
from kitsune.users.tests import ProfileFactory, UserFactory
from kitsune.wiki.tests import RevisionFactory
class UserSearchTests(ElasticTestCase):
def test_add_and_delete(self):
"""Adding a user with a profile should add it to the index.
Deleting should delete it.
"""
p = ProfileFactory()
self.refresh()
eq_(UserMappingType.search().count(), 1)
p.user.delete()
self.refresh()
eq_(UserMappingType.search().count(), 0)
def test_data_in_index(self):
"""Verify the data we are indexing."""
u = UserFactory(username="r1cky", email="r@r.com", profile__name="Rick Róss")
self.refresh()
eq_(UserMappingType.search().count(), 1)
data = UserMappingType.search()[0]
eq_(data["username"], u.username)
eq_(data["display_name"], u.profile.name)
u = UserFactory(username="willkg", email="w@w.com", profile__name="Will Cage")
self.refresh()
eq_(UserMappingType.search().count(), 2)
def test_suggest_completions(self):
u1 = UserFactory(username="r1cky", profile__name="Rick Róss")
u2 = UserFactory(username="Willkg", profile__name="Will Cage")
self.refresh()
eq_(UserMappingType.search().count(), 2)
results = UserMappingType.suggest_completions("wi")
eq_(1, len(results))
eq_("Will Cage (Willkg)", results[0]["text"])
eq_(u2.id, results[0]["payload"]["user_id"])
results = UserMappingType.suggest_completions("R1")
eq_(1, len(results))
eq_("Rick Róss (r1cky)", results[0]["text"])
eq_(u1.id, results[0]["payload"]["user_id"])
# Add another Ri....
UserFactory(username="richard", profile__name="Richard Smith")
self.refresh()
eq_(UserMappingType.search().count(), 3)
results = UserMappingType.suggest_completions("ri")
eq_(2, len(results))
texts = [r["text"] for r in results]
assert "Rick Róss (r1cky)" in texts
assert "Richard Smith (richard)" in texts
results = UserMappingType.suggest_completions("Rick Ró")
eq_(1, len(results))
texts = [r["text"] for r in results]
eq_("Rick Róss (r1cky)", results[0]["text"])
def test_suggest_completions_numbers(self):
u1 = UserFactory(username="1337mike", profile__name="Elite Mike")
UserFactory(username="crazypants", profile__name="Crazy Pants")
self.refresh()
eq_(UserMappingType.search().count(), 2)
results = UserMappingType.suggest_completions("13")
eq_(1, len(results))
eq_("Elite Mike (1337mike)", results[0]["text"])
eq_(u1.id, results[0]["payload"]["user_id"])
def test_query_username_with_numbers(self):
u = UserFactory(username="1337miKE", profile__name="Elite Mike")
UserFactory(username="mike", profile__name="NotElite Mike")
self.refresh()
eq_(UserMappingType.search().query(iusername__match="1337mike").count(), 1)
data = UserMappingType.search().query(iusername__match="1337mike")[0]
eq_(data["username"], u.username)
eq_(data["display_name"], u.profile.name)
def test_query_display_name_with_whitespace(self):
UserFactory(username="1337miKE", profile__name="Elite Mike")
UserFactory(username="mike", profile__name="NotElite Mike")
self.refresh()
eq_(UserMappingType.search().count(), 2)
eq_(UserMappingType.search().query(idisplay_name__match_whitespace="elite").count(), 1)
def test_last_contribution_date(self):
"""Verify the last_contribution_date field works properly."""
u = UserFactory(username="satdav")
self.refresh()
data = UserMappingType.search().query(username__match="satdav")[0]
assert not data["last_contribution_date"]
# Add a Support Forum answer. It should be the last contribution.
d = datetime(2014, 1, 2)
AnswerFactory(creator=u, created=d)
u.profile.save() # we need to resave the profile to force a reindex
self.refresh()
data = UserMappingType.search().query(username__match="satdav")[0]
eq_(data["last_contribution_date"], d)
# Add a Revision edit. It should be the last contribution.
d = datetime(2014, 1, 3)
RevisionFactory(created=d, creator=u)
u.profile.save() # we need to resave the profile to force a reindex
self.refresh()
data = UserMappingType.search().query(username__match="satdav")[0]
eq_(data["last_contribution_date"], d)
# Add a Revision review. It should be the last contribution.
d = datetime(2014, 1, 4)
RevisionFactory(reviewed=d, reviewer=u)
u.profile.save() # we need to resave the profile to force a reindex
self.refresh()
data = UserMappingType.search().query(username__match="satdav")[0]
eq_(data["last_contribution_date"], d)
def test_reindex_users_that_contributed_yesterday(self):
yesterday = datetime.now() - timedelta(days=1)
# Verify for answers.
u = UserFactory(username="answerer")
AnswerFactory(creator=u, created=yesterday)
call_command("reindex_users_that_contributed_yesterday")
self.refresh()
data = UserMappingType.search().query(username__match="answerer")[0]
eq_(data["last_contribution_date"].date(), yesterday.date())
# Verify for edits.
u = UserFactory(username="editor")
RevisionFactory(creator=u, created=yesterday)
call_command("reindex_users_that_contributed_yesterday")
self.refresh()
data = UserMappingType.search().query(username__match="editor")[0]
eq_(data["last_contribution_date"].date(), yesterday.date())
# Verify for reviews.
u = UserFactory(username="reviewer")
RevisionFactory(reviewer=u, reviewed=yesterday)
call_command("reindex_users_that_contributed_yesterday")
self.refresh()
data = UserMappingType.search().query(username__match="reviewer")[0]
eq_(data["last_contribution_date"].date(), yesterday.date())

15
scripts/cron.py
Просмотреть файл

@ -222,21 +222,6 @@ def job_report_employee_answers():
call_command("report_employee_answers")
@scheduled_job(
"cron",
month="*",
day="*",
hour="01",
minute="30",
max_instances=1,
coalesce=True,
skip=settings.STAGE,
)
@babis.decorator(ping_after=settings.DMS_REINDEX_USERS_THAT_CONTRIBUTED_YESTERDAY)
def job_reindex_users_that_contributed_yesterday():
call_command("reindex_users_that_contributed_yesterday")
@scheduled_job(
"cron",
month="*",