Renamed AddonsCoinstallCache to TAARCache

This commit is contained in:
Victor Ng 2020-09-01 20:20:16 -04:00
Родитель a38083bee5
Коммит ad0f52d429
16 изменённых файлов: 105 добавлений и 127 удалений

48
bin/pipstrap.py Executable file → Normal file
Просмотреть файл

@ -28,6 +28,7 @@ from shutil import rmtree
from subprocess import check_output
from sys import exit
from tempfile import mkdtemp
try:
from urllib2 import build_opener, HTTPHandler, HTTPSHandler
except ImportError:
@ -40,26 +41,34 @@ except ImportError:
PACKAGES = [
# Pip has no dependencies, as it vendors everything:
('https://pypi.python.org/packages/source/p/pip/pip-8.0.2.tar.gz',
'46f4bd0d8dfd51125a554568d646fe4200a3c2c6c36b9f2d06d2212148439521'),
(
"https://pypi.python.org/packages/source/p/pip/pip-8.0.2.tar.gz",
"46f4bd0d8dfd51125a554568d646fe4200a3c2c6c36b9f2d06d2212148439521",
),
# This version of setuptools has only optional dependencies:
('https://pypi.python.org/packages/source/s/setuptools/'
'setuptools-19.4.tar.gz',
'214bf29933f47cf25e6faa569f710731728a07a19cae91ea64f826051f68a8cf'),
(
"https://pypi.python.org/packages/source/s/setuptools/"
"setuptools-19.4.tar.gz",
"214bf29933f47cf25e6faa569f710731728a07a19cae91ea64f826051f68a8cf",
),
# We require Python 2.7 or later because we don't support wheel's
# conditional dep on argparse. This version of wheel has no other
# dependencies:
('https://pypi.python.org/packages/source/w/wheel/wheel-0.26.0.tar.gz',
'eaad353805c180a47545a256e6508835b65a8e830ba1093ed8162f19a50a530c')
(
"https://pypi.python.org/packages/source/w/wheel/wheel-0.26.0.tar.gz",
"eaad353805c180a47545a256e6508835b65a8e830ba1093ed8162f19a50a530c",
),
]
class HashError(Exception):
def __str__(self):
url, path, actual, expected = self.args
return ('{url} did not match the expected hash {expected}. Instead, '
'it was {actual}. The file (left at {path}) may have been '
'tampered with.'.format(**locals()))
return (
"{url} did not match the expected hash {expected}. Instead, "
"it was {actual}. The file (left at {path}) may have been "
"tampered with.".format(**locals())
)
def hashed_download(url, temp, digest):
@ -82,9 +91,9 @@ def hashed_download(url, temp, digest):
yield chunk
response = opener().open(url)
path = join(temp, urlparse(url).path.split('/')[-1])
path = join(temp, urlparse(url).path.split("/")[-1])
actual_hash = sha256()
with open(path, 'wb') as file:
with open(path, "wb") as file:
for chunk in read_chunks(response, 4096):
file.write(chunk)
actual_hash.update(chunk)
@ -96,13 +105,14 @@ def hashed_download(url, temp, digest):
def main():
temp = mkdtemp(prefix='pipstrap-')
temp = mkdtemp(prefix="pipstrap-")
try:
downloads = [hashed_download(url, temp, digest)
for url, digest in PACKAGES]
check_output('pip install --no-index --no-deps -U ' +
' '.join(quote(d) for d in downloads),
shell=True)
downloads = [hashed_download(url, temp, digest) for url, digest in PACKAGES]
check_output(
"pip install --no-index --no-deps -U "
+ " ".join(quote(d) for d in downloads),
shell=True,
)
except HashError as exc:
print(exc)
except Exception:
@ -114,5 +124,5 @@ def main():
return 1
if __name__ == '__main__':
if __name__ == "__main__":
exit(main())

6
bin/taarlite-redis.py → bin/taar-redis.py Executable file → Normal file
Просмотреть файл

@ -1,6 +1,6 @@
#!/usr/bin/env python
from taar.recommenders.redis_cache import AddonsCoinstallCache
from taar.recommenders.redis_cache import TAARCache
from taar.context import default_context
import click
@ -11,7 +11,7 @@ import click
@click.option("--info", is_flag=True, help="Display information about the cache state")
def main(reset, load, info):
"""
Manage the TAARLite redis cache.
Manage the TAAR+TAARLite redis cache.
This expecte that the following enviroment variables are set:
@ -23,7 +23,7 @@ def main(reset, load, info):
return
ctx = default_context()
cache = AddonsCoinstallCache.get_instance(ctx)
cache = TAARCache.get_instance(ctx)
if reset:
if cache.reset():
print("Successfully flushed db0 bookkeeping database.")

Просмотреть файл

@ -8,7 +8,7 @@ import operator as op
from .base_recommender import AbstractRecommender
from taar.recommenders.redis_cache import AddonsCoinstallCache
from taar.recommenders.redis_cache import TAARCache
import markus
@ -40,7 +40,7 @@ class CollaborativeRecommender(AbstractRecommender):
self.logger = self._ctx[IMozLogging].get_logger("taar")
self._redis_cache = AddonsCoinstallCache.get_instance(self._ctx)
self._redis_cache = TAARCache.get_instance(self._ctx)
self.model = None

Просмотреть файл

@ -7,7 +7,7 @@ import itertools
from .base_recommender import AbstractRecommender
from taar.utils import hasher
from taar.recommenders.redis_cache import AddonsCoinstallCache
from taar.recommenders.redis_cache import TAARCache
import markus
@ -32,7 +32,7 @@ class EnsembleRecommender(AbstractRecommender):
self.RECOMMENDER_KEYS = ["collaborative", "similarity", "locale"]
self._ctx = ctx
self._redis_cache = AddonsCoinstallCache.get_instance(self._ctx)
self._redis_cache = TAARCache.get_instance(self._ctx)
self.logger = self._ctx[IMozLogging].get_logger("taar.ensemble")
assert "recommender_factory" in self._ctx

Просмотреть файл

@ -10,7 +10,7 @@ from srgutil.interfaces import IMozLogging
import markus
from taar.recommenders.redis_cache import AddonsCoinstallCache
from taar.recommenders.redis_cache import TAARCache
metrics = markus.get_metrics("taar")
@ -76,7 +76,7 @@ class GuidBasedRecommender:
self._ctx = ctx
self.logger = self._ctx[IMozLogging].get_logger("taarlite")
self._redis_cache = AddonsCoinstallCache.get_instance(self._ctx)
self._redis_cache = TAARCache.get_instance(self._ctx)
self.logger.info("GUIDBasedRecommender is initialized")
def cache_ready(self):

Просмотреть файл

@ -7,7 +7,7 @@ import markus
from srgutil.interfaces import IMozLogging
from .base_recommender import AbstractRecommender
from taar.recommenders.redis_cache import AddonsCoinstallCache
from taar.recommenders.redis_cache import TAARCache
metrics = markus.get_metrics("taar")
@ -28,7 +28,7 @@ class LocaleRecommender(AbstractRecommender):
self.logger = self._ctx[IMozLogging].get_logger("taar")
self._redis_cache = AddonsCoinstallCache.get_instance(self._ctx)
self._redis_cache = TAARCache.get_instance(self._ctx)
# DONE removed
@property

Просмотреть файл

@ -8,7 +8,7 @@ from taar.recommenders.ensemble_recommender import (
)
from taar.recommenders.randomizer import in_experiment, reorder_guids
from srgutil.interfaces import IMozLogging
from taar.recommenders.redis_cache import AddonsCoinstallCache
from taar.recommenders.redis_cache import TAARCache
from taar.settings import TAAR_EXPERIMENT_PROB
@ -57,7 +57,7 @@ class RecommendationManager:
# The whitelist data is only used for test client IDs
self._redis_cache = AddonsCoinstallCache.get_instance(self._ctx)
self._redis_cache = TAARCache.get_instance(self._ctx)
self._experiment_prob = ctx.get("TAAR_EXPERIMENT_PROB", TAAR_EXPERIMENT_PROB)

Просмотреть файл

@ -122,7 +122,7 @@ class PrefixStripper:
return result
class AddonsCoinstallCache:
class TAARCache:
"""
This class manages a redis instance to hold onto the taar-lite
GUID->GUID co-installation data
@ -133,7 +133,7 @@ class AddonsCoinstallCache:
@classmethod
def get_instance(cls, ctx):
if cls._instance is None:
cls._instance = AddonsCoinstallCache(ctx, i_didnt_read_the_docs=False)
cls._instance = TAARCache(ctx, i_didnt_read_the_docs=False)
return cls._instance
def __init__(self, ctx, i_didnt_read_the_docs=True):

Просмотреть файл

@ -7,7 +7,7 @@ from itertools import groupby
from scipy.spatial import distance
from srgutil.interfaces import IMozLogging
import numpy as np
from taar.recommenders.redis_cache import AddonsCoinstallCache
from taar.recommenders.redis_cache import TAARCache
import markus
@ -46,7 +46,7 @@ class SimilarityRecommender(AbstractRecommender):
def __init__(self, ctx):
self._ctx = ctx
self._redis_cache = AddonsCoinstallCache.get_instance(self._ctx)
self._redis_cache = TAARCache.get_instance(self._ctx)
self.logger = self._ctx[IMozLogging].get_logger("taar")

Просмотреть файл

@ -4,19 +4,17 @@ Noop helpers
"""
import mock
from taar.recommenders.redis_cache import AddonsCoinstallCache
from taar.recommenders.redis_cache import TAARCache
def noop_taarlite_dataload(stack):
# no-op the taarlite rankdata
stack.enter_context(
mock.patch.object(AddonsCoinstallCache, "_update_rank_data", return_value=None)
mock.patch.object(TAARCache, "_update_rank_data", return_value=None)
)
# no-op the taarlite guidguid data
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache, "_update_coinstall_data", return_value=None,
)
mock.patch.object(TAARCache, "_update_coinstall_data", return_value=None,)
)
return stack
@ -24,9 +22,7 @@ def noop_taarlite_dataload(stack):
def noop_taarlocale_dataload(stack):
# no-op the taarlite rankdata
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache, "_update_locale_data", return_value=None
)
mock.patch.object(TAARCache, "_update_locale_data", return_value=None)
)
return stack
@ -34,9 +30,7 @@ def noop_taarlocale_dataload(stack):
def noop_taarcollab_dataload(stack):
# no-op the taar collab
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache, "_update_collab_data", return_value=None
)
mock.patch.object(TAARCache, "_update_collab_data", return_value=None)
)
return stack
@ -44,9 +38,7 @@ def noop_taarcollab_dataload(stack):
def noop_taarsimilarity_dataload(stack):
# no-op the taar collab
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache, "_update_similarity_data", return_value=None
)
mock.patch.object(TAARCache, "_update_similarity_data", return_value=None)
)
return stack
@ -54,13 +46,9 @@ def noop_taarsimilarity_dataload(stack):
def noop_taarensemble_dataload(stack):
# no-op the taar collab
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache, "_update_ensemble_data", return_value=None
)
mock.patch.object(TAARCache, "_update_ensemble_data", return_value=None)
)
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache, "_update_whitelist_data", return_value=None
)
mock.patch.object(TAARCache, "_update_whitelist_data", return_value=None)
)
return stack

Просмотреть файл

@ -11,7 +11,7 @@ import numpy
import fakeredis
import mock
import contextlib
from taar.recommenders.redis_cache import AddonsCoinstallCache
from taar.recommenders.redis_cache import TAARCache
from taar.recommenders.collaborative_recommender import CollaborativeRecommender
@ -51,20 +51,16 @@ def mock_install_none_mock_data(ctx):
we always get 404 errors.
"""
with contextlib.ExitStack() as stack:
AddonsCoinstallCache._instance = None
TAARCache._instance = None
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache,
"_fetch_collaborative_item_matrix",
return_value="",
TAARCache, "_fetch_collaborative_item_matrix", return_value="",
)
)
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache,
"_fetch_collaborative_mapping_data",
return_value="",
TAARCache, "_fetch_collaborative_mapping_data", return_value="",
)
)
@ -73,7 +69,7 @@ def mock_install_none_mock_data(ctx):
# Patch fakeredis in
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache,
TAARCache,
"init_redis_connections",
return_value={
0: fakeredis.FakeStrictRedis(db=0),
@ -84,7 +80,7 @@ def mock_install_none_mock_data(ctx):
)
# Initialize redis
AddonsCoinstallCache.get_instance(ctx).safe_load_data()
TAARCache.get_instance(ctx).safe_load_data()
yield stack
@ -113,17 +109,17 @@ def mock_install_mock_data(ctx):
fake_mapping[str(java_hash)] = addon
with contextlib.ExitStack() as stack:
AddonsCoinstallCache._instance = None
TAARCache._instance = None
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache,
TAARCache,
"_fetch_collaborative_item_matrix",
return_value=fake_addon_matrix,
)
)
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache,
TAARCache,
"_fetch_collaborative_mapping_data",
return_value=fake_mapping,
)
@ -134,7 +130,7 @@ def mock_install_mock_data(ctx):
# Patch fakeredis in
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache,
TAARCache,
"init_redis_connections",
return_value={
0: fakeredis.FakeStrictRedis(db=0),
@ -145,7 +141,7 @@ def mock_install_mock_data(ctx):
)
# Initialize redis
AddonsCoinstallCache.get_instance(ctx).safe_load_data()
TAARCache.get_instance(ctx).safe_load_data()
yield stack

Просмотреть файл

@ -6,7 +6,7 @@ from taar.recommenders.ensemble_recommender import EnsembleRecommender
import mock
import contextlib
import fakeredis
from taar.recommenders.redis_cache import AddonsCoinstallCache
from taar.recommenders.redis_cache import TAARCache
from .noop_fixtures import (
noop_taarlocale_dataload,
noop_taarcollab_dataload,
@ -59,16 +59,14 @@ def mock_install_mock_ensemble_data(ctx):
]
with contextlib.ExitStack() as stack:
AddonsCoinstallCache._instance = None
TAARCache._instance = None
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache, "_fetch_ensemble_weights", return_value=DATA,
)
mock.patch.object(TAARCache, "_fetch_ensemble_weights", return_value=DATA,)
)
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache, "_fetch_whitelist", return_value=WHITELIST_DATA,
TAARCache, "_fetch_whitelist", return_value=WHITELIST_DATA,
)
)
@ -77,7 +75,7 @@ def mock_install_mock_ensemble_data(ctx):
# Patch fakeredis in
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache,
TAARCache,
"init_redis_connections",
return_value={
0: fakeredis.FakeStrictRedis(db=0),
@ -88,7 +86,7 @@ def mock_install_mock_ensemble_data(ctx):
)
# Initialize redis
AddonsCoinstallCache.get_instance(ctx).safe_load_data()
TAARCache.get_instance(ctx).safe_load_data()
yield stack

Просмотреть файл

@ -13,7 +13,7 @@ from .noop_fixtures import (
)
from taar.recommenders.guid_based_recommender import GuidBasedRecommender
from taar.recommenders.redis_cache import AddonsCoinstallCache
from taar.recommenders.redis_cache import TAARCache
from taar.recommenders.redis_cache import NORMDATA_GUID_ROW_NORM_PREFIX
@ -92,18 +92,16 @@ RESULTS = {
def mock_coinstall_ranking_context(ctx, mock_coinstall, mock_ranking):
with contextlib.ExitStack() as stack:
AddonsCoinstallCache._instance = None
TAARCache._instance = None
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache, "_fetch_ranking_data", return_value=mock_ranking,
TAARCache, "_fetch_ranking_data", return_value=mock_ranking,
)
)
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache,
"_fetch_coinstall_data",
return_value=mock_coinstall,
TAARCache, "_fetch_coinstall_data", return_value=mock_coinstall,
)
)
@ -115,7 +113,7 @@ def mock_coinstall_ranking_context(ctx, mock_coinstall, mock_ranking):
# Patch fakeredis in
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache,
TAARCache,
"init_redis_connections",
return_value={
0: fakeredis.FakeStrictRedis(db=0),
@ -126,7 +124,7 @@ def mock_coinstall_ranking_context(ctx, mock_coinstall, mock_ranking):
)
# Initialize redis
AddonsCoinstallCache.get_instance(ctx).safe_load_data()
TAARCache.get_instance(ctx).safe_load_data()
yield stack

Просмотреть файл

@ -7,7 +7,7 @@ import mock
import contextlib
import fakeredis
from taar.recommenders.redis_cache import AddonsCoinstallCache
from taar.recommenders.redis_cache import TAARCache
from .noop_fixtures import (
noop_taarcollab_dataload,
noop_taarlite_dataload,
@ -50,12 +50,10 @@ def install_mock_data(ctx):
@contextlib.contextmanager
def mock_locale_data(ctx):
with contextlib.ExitStack() as stack:
AddonsCoinstallCache._instance = None
TAARCache._instance = None
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache,
"_fetch_locale_data",
return_value=FAKE_LOCALE_DATA,
TAARCache, "_fetch_locale_data", return_value=FAKE_LOCALE_DATA,
)
)
@ -67,7 +65,7 @@ def mock_locale_data(ctx):
# Patch fakeredis in
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache,
TAARCache,
"init_redis_connections",
return_value={
0: fakeredis.FakeStrictRedis(db=0),
@ -78,7 +76,7 @@ def mock_locale_data(ctx):
)
# Initialize redis
AddonsCoinstallCache.get_instance(ctx).safe_load_data()
TAARCache.get_instance(ctx).safe_load_data()
yield stack

Просмотреть файл

@ -23,7 +23,7 @@ from markus.testing import MetricsMock
import mock
import contextlib
import fakeredis
from taar.recommenders.redis_cache import AddonsCoinstallCache
from taar.recommenders.redis_cache import TAARCache
@contextlib.contextmanager
@ -37,16 +37,14 @@ def mock_install_mock_curated_data(ctx):
}
with contextlib.ExitStack() as stack:
AddonsCoinstallCache._instance = None
TAARCache._instance = None
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache, "_fetch_whitelist", return_value=mock_data
)
mock.patch.object(TAARCache, "_fetch_whitelist", return_value=mock_data)
)
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache,
TAARCache,
"_fetch_ensemble_weights",
return_value=mock_ensemble_weights,
)
@ -58,15 +56,13 @@ def mock_install_mock_curated_data(ctx):
stack = noop_taarsimilarity_dataload(stack)
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache, "_fetch_whitelist", return_value=mock_data
)
mock.patch.object(TAARCache, "_fetch_whitelist", return_value=mock_data)
)
# Patch fakeredis in
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache,
TAARCache,
"init_redis_connections",
return_value={
0: fakeredis.FakeStrictRedis(db=0),
@ -86,7 +82,7 @@ def mock_install_mock_curated_data(ctx):
ctx["recommender_factory"] = MockRecommenderFactory()
# Initialize redis
AddonsCoinstallCache.get_instance(ctx).safe_load_data()
TAARCache.get_instance(ctx).safe_load_data()
yield stack

Просмотреть файл

@ -31,7 +31,7 @@ from .noop_fixtures import (
noop_taarlocale_dataload,
noop_taarensemble_dataload,
)
from taar.recommenders.redis_cache import AddonsCoinstallCache
from taar.recommenders.redis_cache import TAARCache
def noop_loaders(stack):
@ -82,17 +82,13 @@ def generate_a_fake_taar_client():
def mock_install_no_data(ctx):
with contextlib.ExitStack() as stack:
AddonsCoinstallCache._instance = None
TAARCache._instance = None
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache, "_fetch_similarity_donors", return_value="",
)
mock.patch.object(TAARCache, "_fetch_similarity_donors", return_value="",)
)
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache, "_fetch_similarity_lrcurves", return_value="",
)
mock.patch.object(TAARCache, "_fetch_similarity_lrcurves", return_value="",)
)
stack = noop_loaders(stack)
@ -100,7 +96,7 @@ def mock_install_no_data(ctx):
# Patch fakeredis in
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache,
TAARCache,
"init_redis_connections",
return_value={
0: fakeredis.FakeStrictRedis(db=0),
@ -111,7 +107,7 @@ def mock_install_no_data(ctx):
)
# Initialize redis
AddonsCoinstallCache.get_instance(ctx).safe_load_data()
TAARCache.get_instance(ctx).safe_load_data()
yield stack
@ -119,10 +115,10 @@ def mock_install_no_data(ctx):
def mock_install_categorical_data(ctx):
with contextlib.ExitStack() as stack:
AddonsCoinstallCache._instance = None
TAARCache._instance = None
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache,
TAARCache,
"_fetch_similarity_donors",
return_value=CATEGORICAL_FEATURE_FIXTURE_DATA,
)
@ -130,7 +126,7 @@ def mock_install_categorical_data(ctx):
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache,
TAARCache,
"_fetch_similarity_lrcurves",
return_value=generate_fake_lr_curves(1000),
)
@ -140,7 +136,7 @@ def mock_install_categorical_data(ctx):
# Patch fakeredis in
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache,
TAARCache,
"init_redis_connections",
return_value={
0: fakeredis.FakeStrictRedis(db=0),
@ -151,7 +147,7 @@ def mock_install_categorical_data(ctx):
)
# Initialize redis
AddonsCoinstallCache.get_instance(ctx).safe_load_data()
TAARCache.get_instance(ctx).safe_load_data()
yield stack
@ -161,18 +157,16 @@ def mock_install_continuous_data(ctx):
lrs_data = generate_fake_lr_curves(1000)
with contextlib.ExitStack() as stack:
AddonsCoinstallCache._instance = None
TAARCache._instance = None
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache, "_fetch_similarity_donors", return_value=cts_data,
TAARCache, "_fetch_similarity_donors", return_value=cts_data,
)
)
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache,
"_fetch_similarity_lrcurves",
return_value=lrs_data,
TAARCache, "_fetch_similarity_lrcurves", return_value=lrs_data,
)
)
stack = noop_loaders(stack)
@ -180,7 +174,7 @@ def mock_install_continuous_data(ctx):
# Patch fakeredis in
stack.enter_context(
mock.patch.object(
AddonsCoinstallCache,
TAARCache,
"init_redis_connections",
return_value={
0: fakeredis.FakeStrictRedis(db=0),
@ -191,7 +185,7 @@ def mock_install_continuous_data(ctx):
)
# Initialize redis
AddonsCoinstallCache.get_instance(ctx).safe_load_data()
TAARCache.get_instance(ctx).safe_load_data()
yield stack