Updated all recommenders to use JSON cache

taar/recommenders/collaborative_recommender.py

@@ -35,7 +35,7 @@ class CollaborativeRecommender(AbstractRecommender):
     def __init__(self, ctx):
         self._ctx = ctx
 
-        assert 'utils' in self._ctx
+        assert 'cache' in self._ctx
 
         self._load_json_models()
         self.model = None
@@ -43,11 +43,11 @@ class CollaborativeRecommender(AbstractRecommender):
 
     def _load_json_models(self):
         # Download the addon mappings.
-        self.addon_mapping = self._ctx['utils'].fetch_json(ADDON_MAPPING_URL)
+        self.addon_mapping = self._ctx['cache'].fetch_json(ADDON_MAPPING_URL)
         if self.addon_mapping is None:
             logger.error("Cannot download the addon mapping file {}".format(ADDON_MAPPING_URL))
 
-        self.raw_item_matrix = self._ctx['utils'].fetch_json(ADDON_MODEL_URL)
+        self.raw_item_matrix = self._ctx['cache'].fetch_json(ADDON_MODEL_URL)
         if self.addon_mapping is None:
             logger.error("Cannot download the model file {}".format(ADDON_MODEL_URL))
 

taar/recommenders/ensemble_recommender.py

@@ -17,7 +17,7 @@ logger = logging.getLogger(__name__)
 class WeightCache:
     def __init__(self, ctx):
         self._ctx = ctx
-        assert 'utils' in self._ctx
+        assert 'cache' in self._ctx
 
         self._lock = threading.RLock()
 
@@ -38,7 +38,7 @@ class WeightCache:
                     self._expiry = now + 300
 
             if self._weights is None:
-                tmp = self._ctx['utils'].get_s3_json_content(S3_BUCKET, ENSEMBLE_WEIGHTS)
+                tmp = self._ctx['cache'].get_s3_json_content(S3_BUCKET, ENSEMBLE_WEIGHTS)
                 self._weights = tmp['ensemble_weights']
 
             return self._weights

taar/recommenders/legacy_recommender.py

@@ -19,11 +19,11 @@ class LegacyRecommender(AbstractRecommender):
     """
     def __init__(self, ctx):
         self._ctx = ctx
-        assert 'utils' in self._ctx
+        assert 'cache' in self._ctx
         self._init_from_ctx()
 
     def _init_from_ctx(self):
-        self.legacy_replacements = self._ctx['utils'].get_s3_json_content(ADDON_LIST_BUCKET,
+        self.legacy_replacements = self._ctx['cache'].get_s3_json_content(ADDON_LIST_BUCKET,
                                                                           ADDON_LIST_KEY)
         if self.legacy_replacements is None:
             logger.error("Cannot download the JSON resource: {}".format(ADDON_LIST_KEY))

taar/recommenders/locale_recommender.py

@@ -20,12 +20,12 @@ class LocaleRecommender(AbstractRecommender):
     """
     def __init__(self, ctx):
         self._ctx = ctx
-        assert 'utils' in self._ctx
+        assert 'cache' in self._ctx
         self._init_from_ctx()
 
     def _init_from_ctx(self):
-        utils = self._ctx['utils']
-        self.top_addons_per_locale = utils.get_s3_json_content(ADDON_LIST_BUCKET,
+        cache = self._ctx['cache']
+        self.top_addons_per_locale = cache.get_s3_json_content(ADDON_LIST_BUCKET,
                                                                ADDON_LIST_KEY)
         if self.top_addons_per_locale is None:
             logger.error("Cannot download the top per locale file {}".format(ADDON_LIST_KEY))

taar/recommenders/similarity_recommender.py

@@ -35,19 +35,19 @@ class SimilarityRecommender(AbstractRecommender):
 
     def __init__(self, ctx):
         self._ctx = ctx
-        assert 'utils' in self._ctx
+        assert 'cache' in self._ctx
 
         self._init_from_ctx()
 
     def _init_from_ctx(self):
         # Download the addon donors list.
-        utils = self._ctx['utils']
-        self.donors_pool = utils.get_s3_json_content(S3_BUCKET, DONOR_LIST_KEY)
+        cache = self._ctx['cache']
+        self.donors_pool = cache.get_s3_json_content(S3_BUCKET, DONOR_LIST_KEY)
         if self.donors_pool is None:
             logger.error("Cannot download the donor list: {}".format(DONOR_LIST_KEY))
 
         # Download the probability mapping curves from similarity to likelihood of being a good donor.
-        self.lr_curves = utils.get_s3_json_content(S3_BUCKET, LR_CURVES_SIMILARITY_TO_PROBABILITY)
+        self.lr_curves = cache.get_s3_json_content(S3_BUCKET, LR_CURVES_SIMILARITY_TO_PROBABILITY)
         if self.lr_curves is None:
             logger.error("Cannot download the lr curves: {}".format(LR_CURVES_SIMILARITY_TO_PROBABILITY))
         self.build_features_caches()

tests/test_collaborativerecommender.py

@@ -4,9 +4,12 @@ Test cases for the TAAR CollaborativeRecommender
 
 import numpy
 
+from taar.context import Context
+from taar.cache import JSONCache, Clock
+
 from taar.recommenders.collaborative_recommender import ADDON_MAPPING_URL
 from taar.recommenders.collaborative_recommender import ADDON_MODEL_URL
-from taar.context import Context
+
 
 from taar.recommenders.collaborative_recommender import CollaborativeRecommender
 from taar.recommenders.collaborative_recommender import positive_hash
@@ -57,6 +60,8 @@ def activate_error_responses(ctx):
         def fetch_json(self, url):
             return None
     ctx['utils'] = ErrorUtils()
+    ctx['clock'] = Clock()
+    ctx['cache'] = JSONCache(ctx)
     return ctx
 
 
@@ -77,6 +82,8 @@ def activate_responses(ctx):
                 return FAKE_MAPPING
 
     ctx['utils'] = MockUtils()
+    ctx['clock'] = Clock()
+    ctx['cache'] = JSONCache(ctx)
     return ctx
 
 

tests/test_ensemblerecommender.py

@@ -1,4 +1,6 @@
 from taar.context import Context
+from taar.cache import JSONCache, Clock
+
 from taar.recommenders.ensemble_recommender import WeightCache, EnsembleRecommender
 from .mocks import MockRecommenderFactory
 
@@ -16,6 +18,8 @@ def test_weight_cache():   # noqa
 
     ctx = Context()
     ctx['utils'] = Mocker()
+    ctx['clock'] = Clock()
+    ctx['cache'] = JSONCache(ctx)
 
     wc = WeightCache(ctx.child())
     actual = wc.getWeights()
@@ -26,6 +30,8 @@ def test_recommendations():
     ctx = Context()
 
     ctx['utils'] = Mocker()
+    ctx['clock'] = Clock()
+    ctx['cache'] = JSONCache(ctx)
 
     EXPECTED_RESULTS = [('ghi', 3430.0),
                         ('def', 3320.0),

tests/test_legacyrecommender.py

@@ -1,4 +1,6 @@
 from taar.context import Context
+from taar.cache import JSONCache, Clock
+
 from taar.recommenders import LegacyRecommender
 
 FAKE_LEGACY_DATA = {
@@ -25,6 +27,8 @@ def s3_mocker(ctx):
         def get_s3_json_content(self, *args, **kwargs):
             return FAKE_LEGACY_DATA
     ctx['utils'] = Mocker()
+    ctx['clock'] = Clock()
+    ctx['cache'] = JSONCache(ctx)
     return ctx
 
 

tests/test_localerecommender.py

@@ -1,4 +1,6 @@
 from taar.context import Context
+from taar.cache import JSONCache, Clock
+
 from taar.recommenders import LocaleRecommender
 
 
@@ -21,6 +23,8 @@ class MockUtils:
 def create_test_ctx():
     ctx = Context()
     ctx['utils'] = MockUtils()
+    ctx['clock'] = Clock()
+    ctx['cache'] = JSONCache(ctx)
     return ctx.child()
 
 

tests/test_recommendation_manager.py

@@ -1,4 +1,10 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
 from taar.context import Context
+from taar.cache import JSONCache, Clock
+
 from taar.profile_fetcher import ProfileFetcher
 from taar.recommenders import RecommendationManager
 from taar.recommenders.base_recommender import AbstractRecommender
@@ -35,6 +41,8 @@ def get_test_ctx():
 
 def test_none_profile_returns_empty_list():
     ctx = get_test_ctx()
+    ctx['clock'] = Clock()
+    ctx['cache'] = JSONCache(ctx)
     rec_manager = RecommendationManager(ctx)
     assert rec_manager.recommend("random-client-id", 10) == []
 
@@ -64,6 +72,8 @@ def test_recommendation_strategy():
     ctx['recommender_factory'] = factory
     ctx['profile_fetcher'] = StubFetcher()
     ctx['utils'] = Mocker()
+    ctx['clock'] = Clock()
+    ctx['cache'] = JSONCache(ctx)
     manager = RecommendationManager(ctx.child())
     results = manager.recommend("client-id",
                                 10,
@@ -94,6 +104,8 @@ def test_recommendations_via_manager():  # noqa
     ctx['recommender_factory'] = factory
     ctx['profile_fetcher'] = MockProfileFetcher()
     ctx['utils'] = Mocker()
+    ctx['clock'] = Clock()
+    ctx['cache'] = JSONCache(ctx)
     manager = RecommendationManager(ctx.child())
     recommendation_list = manager.recommend({'client_id': 'some_ignored_id'},
                                             10,

tests/test_similarityrecommender.py

@@ -1,3 +1,7 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
 import json
 import six
 
@@ -5,6 +9,8 @@ import numpy as np
 import scipy.stats
 
 from taar.context import Context
+from taar.cache import JSONCache, Clock
+
 from taar.recommenders.similarity_recommender import \
     CATEGORICAL_FEATURES, CONTINUOUS_FEATURES, DONOR_LIST_KEY, LR_CURVES_SIMILARITY_TO_PROBABILITY, \
     SimilarityRecommender
@@ -81,12 +87,16 @@ class MockContinuousData:
 def create_cat_test_ctx():
     ctx = Context()
     ctx['utils'] = MockCategoricalData()
+    ctx['clock'] = Clock()
+    ctx['cache'] = JSONCache(ctx)
     return ctx.child()
 
 
 def create_cts_test_ctx():
     ctx = Context()
     ctx['utils'] = MockContinuousData()
+    ctx['clock'] = Clock()
+    ctx['cache'] = JSONCache(ctx)
     return ctx.child()
 
 
@@ -94,6 +104,8 @@ def test_soft_fail():
     # Create a new instance of a SimilarityRecommender.
     ctx = Context()
     ctx['utils'] = MockNoDataUtils()
+    ctx['clock'] = Clock()
+    ctx['cache'] = JSONCache(ctx)
     r = SimilarityRecommender(ctx)
 
     # Don't recommend if the source files cannot be found.