diff --git a/k8s/regions/frankfurt/dev.yaml b/k8s/regions/frankfurt/dev.yaml index 45928cb45..a7b6794a5 100644 --- a/k8s/regions/frankfurt/dev.yaml +++ b/k8s/regions/frankfurt/dev.yaml @@ -71,7 +71,6 @@ app: dms_rebuild_kb: SECRET dms_reindex: SECRET dms_reindex_es7: SECRET - dms_reindex_users_that_contributed_yesterday: SECRET dms_reload_question_traffic_stats: SECRET dms_reload_wiki_traffic_stats: SECRET dms_report_employee_answers: SECRET diff --git a/k8s/regions/frankfurt/prod.yaml b/k8s/regions/frankfurt/prod.yaml index c184b8794..0aca87c5b 100644 --- a/k8s/regions/frankfurt/prod.yaml +++ b/k8s/regions/frankfurt/prod.yaml @@ -69,7 +69,6 @@ app: dms_rebuild_kb: SECRET dms_reindex: SECRET dms_reindex_es7: SECRET - dms_reindex_users_that_contributed_yesterday: SECRET dms_reload_question_traffic_stats: SECRET dms_reload_wiki_traffic_stats: SECRET dms_report_employee_answers: SECRET diff --git a/k8s/regions/frankfurt/stage.yaml b/k8s/regions/frankfurt/stage.yaml index ff5d0ba9e..fa97bc955 100644 --- a/k8s/regions/frankfurt/stage.yaml +++ b/k8s/regions/frankfurt/stage.yaml @@ -72,7 +72,6 @@ app: dms_rebuild_kb: SECRET dms_reindex: SECRET dms_reindex_es7: SECRET - dms_reindex_users_that_contributed_yesterday: SECRET dms_reload_question_traffic_stats: SECRET dms_reload_wiki_traffic_stats: SECRET dms_report_employee_answers: SECRET diff --git a/k8s/regions/frankfurt/test.yaml b/k8s/regions/frankfurt/test.yaml index cdffa9398..6f4e8c2a9 100644 --- a/k8s/regions/frankfurt/test.yaml +++ b/k8s/regions/frankfurt/test.yaml @@ -71,7 +71,6 @@ app: dms_rebuild_kb: SECRET dms_reindex: SECRET dms_reindex_es7: SECRET - dms_reindex_users_that_contributed_yesterday: SECRET dms_reload_question_traffic_stats: SECRET dms_reload_wiki_traffic_stats: SECRET dms_report_employee_answers: SECRET diff --git a/k8s/regions/oregon-a/prod.yaml b/k8s/regions/oregon-a/prod.yaml index ee27176bd..b67d4bc37 100644 --- a/k8s/regions/oregon-a/prod.yaml +++ b/k8s/regions/oregon-a/prod.yaml @@ -73,7 +73,6 @@ app: dms_rebuild_kb: SECRET dms_reindex: SECRET dms_reindex_es7: SECRET - dms_reindex_users_that_contributed_yesterday: SECRET dms_reload_question_traffic_stats: SECRET dms_reload_wiki_traffic_stats: SECRET dms_report_employee_answers: SECRET diff --git a/k8s/regions/oregon-a/stage.yaml b/k8s/regions/oregon-a/stage.yaml index d778c1bff..9d7cb01bb 100644 --- a/k8s/regions/oregon-a/stage.yaml +++ b/k8s/regions/oregon-a/stage.yaml @@ -72,7 +72,6 @@ app: dms_rebuild_kb: SECRET dms_reindex: SECRET dms_reindex_es7: SECRET - dms_reindex_users_that_contributed_yesterday: SECRET dms_reload_question_traffic_stats: SECRET dms_reload_wiki_traffic_stats: SECRET dms_report_employee_answers: SECRET diff --git a/k8s/regions/oregon-b/prod.yaml b/k8s/regions/oregon-b/prod.yaml index 62caeca67..daced9a69 100644 --- a/k8s/regions/oregon-b/prod.yaml +++ b/k8s/regions/oregon-b/prod.yaml @@ -73,7 +73,6 @@ app: dms_rebuild_kb: SECRET dms_reindex: SECRET dms_reindex_es7: SECRET - dms_reindex_users_that_contributed_yesterday: SECRET dms_reload_question_traffic_stats: SECRET dms_reload_wiki_traffic_stats: SECRET dms_report_employee_answers: SECRET diff --git a/k8s/regions/oregon-b/stage.yaml b/k8s/regions/oregon-b/stage.yaml index 7128ccba2..f6ccf0484 100644 --- a/k8s/regions/oregon-b/stage.yaml +++ b/k8s/regions/oregon-b/stage.yaml @@ -72,7 +72,6 @@ app: dms_rebuild_kb: SECRET dms_reindex: SECRET dms_reindex_es7: SECRET - dms_reindex_users_that_contributed_yesterday: SECRET dms_reload_question_traffic_stats: SECRET dms_reload_wiki_traffic_stats: SECRET dms_report_employee_answers: SECRET diff --git a/kitsune/settings.py b/kitsune/settings.py index b17b519f1..607f40f01 100644 --- a/kitsune/settings.py +++ b/kitsune/settings.py @@ -1141,9 +1141,6 @@ DMS_UPDATE_TOP_CONTRIBUTORS = config("DMS_UPDATE_TOP_CONTRIBUTORS", default=None DMS_UPDATE_L10N_COVERAGE_METRICS = config("DMS_UPDATE_L10N_COVERAGE_METRICS", default=None) DMS_CALCULATE_CSAT_METRICS = config("DMS_CALCULATE_CSAT_METRICS", default=None) DMS_REPORT_EMPLOYEE_ANSWERS = config("DMS_REPORT_EMPLOYEE_ANSWERS", default=None) -DMS_REINDEX_USERS_THAT_CONTRIBUTED_YESTERDAY = config( - "DMS_REINDEX_USERS_THAT_CONTRIBUTED_YESTERDAY", default=None -) DMS_UPDATE_WEEKLY_VOTES = config("DMS_UPDATE_WEEKLY_VOTES", default=None) DMS_UPDATE_SEARCH_CTR_METRIC = config("DMS_UPDATE_SEARCH_CTR_METRIC", default=None) DMS_UPDATE_CONTRIBUTOR_METRICS = config("DMS_UPDATE_CONTRIBUTOR_METRICS", default=None) diff --git a/kitsune/users/management/__init__.py b/kitsune/users/management/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/kitsune/users/management/commands/__init__.py b/kitsune/users/management/commands/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/kitsune/users/management/commands/reindex_users_that_contributed_yesterday.py b/kitsune/users/management/commands/reindex_users_that_contributed_yesterday.py deleted file mode 100644 index 98bee0ede..000000000 --- a/kitsune/users/management/commands/reindex_users_that_contributed_yesterday.py +++ /dev/null @@ -1,43 +0,0 @@ -from datetime import datetime, timedelta - -from django.core.management.base import BaseCommand - -from kitsune.questions.models import Answer -from kitsune.search.tasks import index_task -from kitsune.search.utils import to_class_path -from kitsune.users.models import UserMappingType -from kitsune.wiki.models import Revision - - -class Command(BaseCommand): - help = "Update the users (in ES) that contributed yesterday." - - def handle(self, **options): - """ - The idea is to update the last_contribution_date field. - """ - today = datetime.now() - yesterday = today - timedelta(days=1) - - # Support Forum answers - user_ids = list( - Answer.objects.filter(created__gte=yesterday, created__lt=today).values_list( - "creator_id", flat=True - ) - ) - - # KB Edits - user_ids += list( - Revision.objects.filter(created__gte=yesterday, created__lt=today).values_list( - "creator_id", flat=True - ) - ) - - # KB Reviews - user_ids += list( - Revision.objects.filter(reviewed__gte=yesterday, reviewed__lt=today).values_list( - "reviewer_id", flat=True - ) - ) - - index_task.delay(to_class_path(UserMappingType), list(set(user_ids))) diff --git a/kitsune/users/models.py b/kitsune/users/models.py index cd2fc4f67..854842a37 100644 --- a/kitsune/users/models.py +++ b/kitsune/users/models.py @@ -1,24 +1,15 @@ import logging import re -import time from datetime import datetime from django.conf import settings from django.contrib.auth.models import User from django.db import models -from django.utils.translation import ugettext as _ from django.utils.translation import ugettext_lazy as _lazy from timezone_field import TimeZoneField from kitsune.lib.countries import COUNTRIES from kitsune.products.models import Product -from kitsune.search.es_utils import UnindexMeBro -from kitsune.search.models import ( - SearchMappingType, - SearchMixin, - register_for_indexing, - register_mapping_type, -) from kitsune.sumo.models import LocaleField, ModelBase from kitsune.sumo.urlresolvers import reverse from kitsune.sumo.utils import auto_delete_files @@ -33,7 +24,7 @@ SET_ID_PREFIX = "https://schemas.accounts.firefox.com/event/" @auto_delete_files -class Profile(ModelBase, SearchMixin): +class Profile(ModelBase): """Profile model for django users.""" user = models.OneToOneField( @@ -147,10 +138,6 @@ class Profile(ModelBase, SearchMixin): def display_name(self): return self.name if self.name else self.user.username - @classmethod - def get_mapping_type(cls): - return UserMappingType - @classmethod def get_serializer(cls, serializer_type="full"): # Avoid circular import @@ -210,112 +197,6 @@ class Profile(ModelBase, SearchMixin): return AnswerVote.objects.filter(answer__creator=self.user, helpful=True).count() -@register_mapping_type -class UserMappingType(SearchMappingType): - list_keys = [ - "twitter_usernames", - "itwitter_usernames", - ] - - @classmethod - def get_model(cls): - return Profile - - @classmethod - def get_index_group(cls): - return "non-critical" - - @classmethod - def get_mapping(cls): - return { - "properties": { - "id": {"type": "long"}, - "model": {"type": "string", "index": "not_analyzed"}, - "url": {"type": "string", "index": "not_analyzed"}, - "indexed_on": {"type": "integer"}, - "username": {"type": "string", "index": "not_analyzed"}, - "display_name": {"type": "string", "index": "not_analyzed"}, - "twitter_usernames": {"type": "string", "index": "not_analyzed"}, - "last_contribution_date": {"type": "date"}, - # lower-cased versions for querying: - "iusername": {"type": "string", "index": "not_analyzed"}, - "idisplay_name": {"type": "string", "analyzer": "whitespace"}, - "itwitter_usernames": {"type": "string", "index": "not_analyzed"}, - "avatar": {"type": "string", "index": "not_analyzed"}, - "suggest": {"type": "completion", "analyzer": "whitespace", "payloads": True}, - } - } - - @classmethod - def extract_document(cls, obj_id, obj=None): - """Extracts interesting thing from a Thread and its Posts""" - if obj is None: - model = cls.get_model() - obj = model.objects.select_related("user").get(pk=obj_id) - - if not obj.user.is_active: - raise UnindexMeBro() - - d = {} - d["id"] = obj.pk - d["model"] = cls.get_mapping_type_name() - d["url"] = obj.get_absolute_url() - d["indexed_on"] = int(time.time()) - - d["username"] = obj.user.username - d["display_name"] = obj.display_name - d["twitter_usernames"] = [] - - d["last_contribution_date"] = obj.last_contribution_date - - d["iusername"] = obj.user.username.lower() - d["idisplay_name"] = obj.display_name.lower() - d["itwitter_usernames"] = [] - - from kitsune.users.templatetags.jinja_helpers import profile_avatar - - d["avatar"] = profile_avatar(obj.user, size=120) - - d["suggest"] = { - "input": [d["iusername"], d["idisplay_name"]], - "output": _("{displayname} ({username})").format( - displayname=d["display_name"], username=d["username"] - ), - "payload": {"user_id": d["id"]}, - } - - return d - - @classmethod - def suggest_completions(cls, text): - """Suggest completions for the text provided.""" - USER_SUGGEST = "user-suggest" - es = UserMappingType.search().get_es() - - results = es.suggest( - index=cls.get_index(), - body={USER_SUGGEST: {"text": text.lower(), "completion": {"field": "suggest"}}}, - ) - - if results[USER_SUGGEST][0]["length"] > 0: - return results[USER_SUGGEST][0]["options"] - - return [] - - -register_for_indexing("users", Profile) - - -def get_profile(u): - try: - return Profile.objects.get(user=u) - except Profile.DoesNotExist: - return None - - -register_for_indexing("users", User, instance_to_indexee=get_profile) - - class Setting(ModelBase): """User specific value per setting""" diff --git a/kitsune/users/tests/test_es.py b/kitsune/users/tests/test_es.py deleted file mode 100644 index 29a4699fb..000000000 --- a/kitsune/users/tests/test_es.py +++ /dev/null @@ -1,176 +0,0 @@ -# -*- coding: utf-8 -*- -from datetime import datetime, timedelta - -from django.core.management import call_command -from nose.tools import eq_ - -from kitsune.questions.tests import AnswerFactory -from kitsune.search.tests.test_es import ElasticTestCase -from kitsune.users.models import UserMappingType -from kitsune.users.tests import ProfileFactory, UserFactory -from kitsune.wiki.tests import RevisionFactory - - -class UserSearchTests(ElasticTestCase): - def test_add_and_delete(self): - """Adding a user with a profile should add it to the index. - - Deleting should delete it. - """ - p = ProfileFactory() - self.refresh() - eq_(UserMappingType.search().count(), 1) - - p.user.delete() - self.refresh() - eq_(UserMappingType.search().count(), 0) - - def test_data_in_index(self): - """Verify the data we are indexing.""" - u = UserFactory(username="r1cky", email="r@r.com", profile__name="Rick Róss") - - self.refresh() - - eq_(UserMappingType.search().count(), 1) - data = UserMappingType.search()[0] - eq_(data["username"], u.username) - eq_(data["display_name"], u.profile.name) - - u = UserFactory(username="willkg", email="w@w.com", profile__name="Will Cage") - self.refresh() - eq_(UserMappingType.search().count(), 2) - - def test_suggest_completions(self): - u1 = UserFactory(username="r1cky", profile__name="Rick Róss") - u2 = UserFactory(username="Willkg", profile__name="Will Cage") - - self.refresh() - eq_(UserMappingType.search().count(), 2) - - results = UserMappingType.suggest_completions("wi") - eq_(1, len(results)) - eq_("Will Cage (Willkg)", results[0]["text"]) - eq_(u2.id, results[0]["payload"]["user_id"]) - - results = UserMappingType.suggest_completions("R1") - eq_(1, len(results)) - eq_("Rick Róss (r1cky)", results[0]["text"]) - eq_(u1.id, results[0]["payload"]["user_id"]) - - # Add another Ri.... - UserFactory(username="richard", profile__name="Richard Smith") - - self.refresh() - eq_(UserMappingType.search().count(), 3) - - results = UserMappingType.suggest_completions("ri") - eq_(2, len(results)) - texts = [r["text"] for r in results] - assert "Rick Róss (r1cky)" in texts - assert "Richard Smith (richard)" in texts - - results = UserMappingType.suggest_completions("Rick Ró") - eq_(1, len(results)) - texts = [r["text"] for r in results] - eq_("Rick Róss (r1cky)", results[0]["text"]) - - def test_suggest_completions_numbers(self): - u1 = UserFactory(username="1337mike", profile__name="Elite Mike") - UserFactory(username="crazypants", profile__name="Crazy Pants") - - self.refresh() - eq_(UserMappingType.search().count(), 2) - - results = UserMappingType.suggest_completions("13") - eq_(1, len(results)) - eq_("Elite Mike (1337mike)", results[0]["text"]) - eq_(u1.id, results[0]["payload"]["user_id"]) - - def test_query_username_with_numbers(self): - u = UserFactory(username="1337miKE", profile__name="Elite Mike") - UserFactory(username="mike", profile__name="NotElite Mike") - - self.refresh() - - eq_(UserMappingType.search().query(iusername__match="1337mike").count(), 1) - data = UserMappingType.search().query(iusername__match="1337mike")[0] - eq_(data["username"], u.username) - eq_(data["display_name"], u.profile.name) - - def test_query_display_name_with_whitespace(self): - UserFactory(username="1337miKE", profile__name="Elite Mike") - UserFactory(username="mike", profile__name="NotElite Mike") - - self.refresh() - - eq_(UserMappingType.search().count(), 2) - eq_(UserMappingType.search().query(idisplay_name__match_whitespace="elite").count(), 1) - - def test_last_contribution_date(self): - """Verify the last_contribution_date field works properly.""" - u = UserFactory(username="satdav") - self.refresh() - - data = UserMappingType.search().query(username__match="satdav")[0] - assert not data["last_contribution_date"] - - # Add a Support Forum answer. It should be the last contribution. - d = datetime(2014, 1, 2) - AnswerFactory(creator=u, created=d) - u.profile.save() # we need to resave the profile to force a reindex - self.refresh() - - data = UserMappingType.search().query(username__match="satdav")[0] - eq_(data["last_contribution_date"], d) - - # Add a Revision edit. It should be the last contribution. - d = datetime(2014, 1, 3) - RevisionFactory(created=d, creator=u) - u.profile.save() # we need to resave the profile to force a reindex - self.refresh() - - data = UserMappingType.search().query(username__match="satdav")[0] - eq_(data["last_contribution_date"], d) - - # Add a Revision review. It should be the last contribution. - d = datetime(2014, 1, 4) - RevisionFactory(reviewed=d, reviewer=u) - - u.profile.save() # we need to resave the profile to force a reindex - self.refresh() - - data = UserMappingType.search().query(username__match="satdav")[0] - eq_(data["last_contribution_date"], d) - - def test_reindex_users_that_contributed_yesterday(self): - yesterday = datetime.now() - timedelta(days=1) - - # Verify for answers. - u = UserFactory(username="answerer") - AnswerFactory(creator=u, created=yesterday) - - call_command("reindex_users_that_contributed_yesterday") - self.refresh() - - data = UserMappingType.search().query(username__match="answerer")[0] - eq_(data["last_contribution_date"].date(), yesterday.date()) - - # Verify for edits. - u = UserFactory(username="editor") - RevisionFactory(creator=u, created=yesterday) - - call_command("reindex_users_that_contributed_yesterday") - self.refresh() - - data = UserMappingType.search().query(username__match="editor")[0] - eq_(data["last_contribution_date"].date(), yesterday.date()) - - # Verify for reviews. - u = UserFactory(username="reviewer") - RevisionFactory(reviewer=u, reviewed=yesterday) - - call_command("reindex_users_that_contributed_yesterday") - self.refresh() - - data = UserMappingType.search().query(username__match="reviewer")[0] - eq_(data["last_contribution_date"].date(), yesterday.date()) diff --git a/scripts/cron.py b/scripts/cron.py index 9be443b3f..1eb440ea5 100644 --- a/scripts/cron.py +++ b/scripts/cron.py @@ -222,21 +222,6 @@ def job_report_employee_answers(): call_command("report_employee_answers") -@scheduled_job( - "cron", - month="*", - day="*", - hour="01", - minute="30", - max_instances=1, - coalesce=True, - skip=settings.STAGE, -) -@babis.decorator(ping_after=settings.DMS_REINDEX_USERS_THAT_CONTRIBUTED_YESTERDAY) -def job_reindex_users_that_contributed_yesterday(): - call_command("reindex_users_that_contributed_yesterday") - - @scheduled_job( "cron", month="*",