From fa6749b5b8954c7f367603b0814a18333975930c Mon Sep 17 00:00:00 2001 From: Chuck Harmston Date: Thu, 25 Feb 2016 10:29:43 -0700 Subject: [PATCH 1/4] Refactors into single-container application with supplementary services in compose (closes #51). --- .dockerignore | 9 +++ Dockerfile | 18 ++++++ app/Dockerfile | 19 ------ conf/{nginx => }/contribute.json | 0 conf/nginx/.dockerignore | 1 - conf/nginx/Dockerfile | 12 ---- conf/nginx/recommendation.conf | 16 ----- conf/web.sh | 7 +++ conf/worker.sh | 3 + docker-compose.yml | 39 +++++------- docs/local.md | 76 +++++++++++++++++++----- recommendation/celery.py | 4 ++ recommendation/conf.py | 17 ++++++ recommendation/tests/test_celery.py | 19 ++++++ recommendation/tests/test_wsgi.py | 24 ++++++++ recommendation/wsgi.py | 4 ++ app/requirements.txt => requirements.txt | 0 server.py | 12 ---- 18 files changed, 180 insertions(+), 100 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile delete mode 100644 app/Dockerfile rename conf/{nginx => }/contribute.json (100%) delete mode 100644 conf/nginx/.dockerignore delete mode 100644 conf/nginx/Dockerfile delete mode 100644 conf/nginx/recommendation.conf create mode 100755 conf/web.sh create mode 100755 conf/worker.sh create mode 100644 recommendation/celery.py create mode 100644 recommendation/conf.py create mode 100644 recommendation/tests/test_celery.py create mode 100644 recommendation/tests/test_wsgi.py create mode 100644 recommendation/wsgi.py rename app/requirements.txt => requirements.txt (100%) delete mode 100644 server.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..c83317b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +__pycache__ +*.pyc +src +.DS_Store +.eggs +.git +dist +.coverage +coverage.xml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..cce107a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +# universal-search-recommendation +# +# VERSION 0.1 + +FROM python:3.5 +MAINTAINER https://mail.mozilla.org/listinfo/testpilot-dev + +RUN groupadd --gid 10001 app && \ + useradd --uid 10001 --gid 10001 --shell /usr/sbin/nologin app + +COPY ./requirements.txt /app/requirements.txt +RUN pip install --upgrade --no-cache-dir -r /app/requirements.txt + +COPY . /app +ENV PYTHONPATH $PYTHONPATH:/app + +USER app +ENTRYPOINT /app/conf/web.sh diff --git a/app/Dockerfile b/app/Dockerfile deleted file mode 100644 index 5d4d4ea..0000000 --- a/app/Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -# universal-search-recommendation-app -# -# VERSION 0.1 - -FROM python:3.5 -MAINTAINER https://mail.mozilla.org/listinfo/testpilot-dev - -RUN groupadd --gid 1001 app && \ - useradd --uid 1001 --gid 1001 --shell /usr/sbin/nologin app - -WORKDIR /app - -COPY ./requirements.txt /app/requirements.txt -RUN pip install --upgrade --no-cache-dir -r requirements.txt - -COPY . /app -ENV PYTHONPATH $PYTHONPATH:/:/app - -USER app diff --git a/conf/nginx/contribute.json b/conf/contribute.json similarity index 100% rename from conf/nginx/contribute.json rename to conf/contribute.json diff --git a/conf/nginx/.dockerignore b/conf/nginx/.dockerignore deleted file mode 100644 index e43b0f9..0000000 --- a/conf/nginx/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -.DS_Store diff --git a/conf/nginx/Dockerfile b/conf/nginx/Dockerfile deleted file mode 100644 index 6633338..0000000 --- a/conf/nginx/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -# universal-search-recommendation-proxy -# -# VERSION 0.1 - -FROM nginx:1.9.9 -MAINTAINER https://mail.mozilla.org/listinfo/testpilot-dev - -RUN rm /etc/nginx/conf.d/default.conf -COPY recommendation.conf /etc/nginx/conf.d/ - -RUN mkdir -p /var/www/ -COPY contribute.json /var/www/ diff --git a/conf/nginx/recommendation.conf b/conf/nginx/recommendation.conf deleted file mode 100644 index 3a72140..0000000 --- a/conf/nginx/recommendation.conf +++ /dev/null @@ -1,16 +0,0 @@ -server { - listen 80; - server_name ""; - charset utf-8; - - location /contribute.json { - alias /var/www/contribute.json; - } - - location / { - proxy_pass http://app:8000; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } -} diff --git a/conf/web.sh b/conf/web.sh new file mode 100755 index 0000000..fc61b25 --- /dev/null +++ b/conf/web.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +<<<<<<< HEAD +uwsgi --http :${PORT:-8000} --wsgi-file ../app/main.py --master +======= +uwsgi --http :${PORT:-8000} --wsgi-file /app/recommendation/wsgi.py --master +>>>>>>> 579c385... fixup! WIP diff --git a/conf/worker.sh b/conf/worker.sh new file mode 100755 index 0000000..337e11f --- /dev/null +++ b/conf/worker.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +celery worker --app=recommendation diff --git a/docker-compose.yml b/docker-compose.yml index 560ced2..fad1446 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,37 +1,26 @@ -app: - build: ./app - command: uwsgi --http :8000 --wsgi-file /app/main.py --master - env_file: .env - expose: - - "8000" - links: - - memcached - - redis - restart: always - -nginx: - build: ./conf/nginx - links: - - app - ports: - - "80:80" - restart: always - memcached: - image: memcached - restart: always expose: - "11211" + image: memcached + restart: always + ports: + - "11211" + - "11211:11211" redis: + expose: + - "6379" image: redis + ports: + - "6379" + - "6379:6379" -celery: - build: ./app - command: celery worker --app=app.main +worker: + build: ./ + entrypoint: /app/conf/worker.sh env_file: .env environment: - - CELERY_BROKER_URL=redis://redis:6379/0 + - RECOMMENDATION_ENV="worker" links: - memcached - redis diff --git a/docs/local.md b/docs/local.md index 7f457bb..288ffb2 100644 --- a/docs/local.md +++ b/docs/local.md @@ -1,40 +1,86 @@ -# Local Development +## Local Development A multiple-container Docker configuration is maintained to support local development. -## First-time setup +### Configuration -A few steps are required the first time +Some environment variables are required. Each of those are contained within `.env.dist`, which should be copied to `.env` and populated as appropriate. + + +### Services + +`universal-search-recommendation` requires several additional services: a [Memcached](http://memcached.org/) cache backend, a [Celery](http://www.celeryproject.org/) task queue, and a [Redis](http://redis.io/) Celery backend. A [Compose](https://docs.docker.com/compose/) configuration is included to streamline setup and management of those services. + + +#### Set up Docker host. + +The first time you set up the services, you must create the Docker host. - [Install Docker Toolbox](https://www.docker.com/products/docker-toolbox). -- Some environment variables are required. Each of those are contained within `.env.dist`, which should be copied to `.env` and populated as appropriate. - Create a Docker host: ```bash -docker-machine create --driver virtualbox universal-search +docker-machine create --driver virtualbox universal-search-dev ``` -- Add an entry to your hosts file: +- Add an entry to your `hosts` file: ```bash -sudo sh -c "echo $(docker-machine ip universal-search 2>/dev/null) universal-search.dev >> /etc/hosts" +sudo sh -c "echo $(docker-machine ip universal-search-dev 2>/dev/null) universal-search.dev >> /etc/hosts" ``` -## Start machine and containers +#### Start services. -The machine and containers can then be started at any time: +After the host is set up, you can run each service by: + +``` +docker-machine start universal-search-dev +eval $(docker-machine env universal-search-dev) +docker-compose up -d +``` + +To verify that each service is up, run: ```bash -docker-machine start universal-search -eval $(docker-machine env universal-search) -docker-compose up +docker-compose ps ``` -While `docker-compose up` is running, the recommendation server may be accessed at [http://universal-search.dev](http://universal-search.dev). +In the output you should see containers named `recommendation_memcached_1`, `recommendation_redis_1`, and `recommendation_worker_1`. The state for each of these should be `Up`. -## Architecture +### Application -An nginx server is the entry point, acting as a reverse proxy to forward requests to a Python container running uWSGI, which manages the application server. Memcached, Celery, and Redis containers run parallel; used for caching, as a task queue, and as a backend for the task queue, respectively. +`universal-search-recommendation` is managed as a Docker container that is separately built from the services, but run on the same host as the services. + +To build it: + +```bash +docker build -t universal-search-recommendation . +``` + +To run it: + +```bash +docker run -d -e "RECOMMENDATION_SERVICES=`docker-machine ip universal-search-dev`" -p 80:8000/tcp universal-search-recommendation +``` + +To verify that the application is running: + +```bash +docker ps --filter ancestor="universal-search-recommendation" +``` + +You should see one container running, and its status should begin with `Up`. + + +### Usage + +If the application and services are all running, you should be able to access to access the recommendation server at [http://universal-search.dev](http://universal-search.dev). After making changes to the application, you will need to stop it: + +```bash +docker stop $(docker ps --filter ancestor="universal-search-recommendation" --format="{{.ID}}") +``` + +Then rebuild and start it back up, per above. diff --git a/recommendation/celery.py b/recommendation/celery.py new file mode 100644 index 0000000..dfd6edd --- /dev/null +++ b/recommendation/celery.py @@ -0,0 +1,4 @@ +from recommendation import tasks # noqa +from recommendation.factory import create_queue + +celery = create_queue() diff --git a/recommendation/conf.py b/recommendation/conf.py new file mode 100644 index 0000000..559bd43 --- /dev/null +++ b/recommendation/conf.py @@ -0,0 +1,17 @@ +from os import environ as env + + +DEBUG = env.get('RECOMMENDATION_ENV', 'development') == 'development' +KEY_PREFIX = env.get('RECOMMENDATION_KEY_PREFIX', 'query_') + +EMBEDLY_API_KEY = env.get('EMBEDLY_API_KEY', '') +YAHOO_OAUTH_KEY = env.get('YAHOO_OAUTH_KEY', '') +YAHOO_OAUTH_SECRET = env.get('YAHOO_OAUTH_SECRET', '') + +RECOMMENDATION_SERVICES = env.get('RECOMMENDATION_SERVICES') +if DEBUG and RECOMMENDATION_SERVICES: + CELERY_BROKER_URL = 'redis://%s:6379/0' % RECOMMENDATION_SERVICES + MEMCACHED_HOST = '%s:11211' % RECOMMENDATION_SERVICES +else: + CELERY_BROKER_URL = env.get('CELERY_BROKER_URL', 'redis://redis:6379/0') + MEMCACHED_HOST = env.get('MEMCACHED_HOST', 'memcached:11211') diff --git a/recommendation/tests/test_celery.py b/recommendation/tests/test_celery.py new file mode 100644 index 0000000..1cf1a0f --- /dev/null +++ b/recommendation/tests/test_celery.py @@ -0,0 +1,19 @@ +from unittest import TestCase + +from celery.app.base import Celery +from nose.tools import eq_, ok_ + +from recommendation.celery import celery + + +class TestCeleryApp(TestCase): + def test_celery_app(self): + from recommendation import celery as celery_module + ok_(hasattr(celery_module, 'celery')) + + def test_celery_app_type(self): + eq_(type(celery), Celery) + + def test_celery_app_singleton(self): + from recommendation.celery import celery as celery_2 + eq_(id(celery), id(celery_2)) diff --git a/recommendation/tests/test_wsgi.py b/recommendation/tests/test_wsgi.py new file mode 100644 index 0000000..ee674c2 --- /dev/null +++ b/recommendation/tests/test_wsgi.py @@ -0,0 +1,24 @@ +from unittest import TestCase + +from flask.app import Flask +from nose.tools import eq_, ok_ + + +class TestCeleryApp(TestCase): + def test_wsgi_app(self): + from recommendation import wsgi as wsgi_module + ok_(hasattr(wsgi_module, 'application')) + ok_(hasattr(wsgi_module, 'celery')) + + def test_wsgi_app_type(self): + from recommendation.wsgi import application + eq_(type(application), Flask) + + def test_wsgi_app_singleton(self): + """ + Tests that multiple imports of the application singleton have the same + memory ID (i.e. is the same object). + """ + from recommendation.wsgi import application as app1 + from recommendation.wsgi import application as app2 + eq_(id(app1), id(app2)) diff --git a/recommendation/wsgi.py b/recommendation/wsgi.py new file mode 100644 index 0000000..aba6372 --- /dev/null +++ b/recommendation/wsgi.py @@ -0,0 +1,4 @@ +from recommendation.celery import celery # noqa +from recommendation.factory import create_app + +application = create_app() diff --git a/app/requirements.txt b/requirements.txt similarity index 100% rename from app/requirements.txt rename to requirements.txt diff --git a/server.py b/server.py deleted file mode 100644 index 8593903..0000000 --- a/server.py +++ /dev/null @@ -1,12 +0,0 @@ -import os -import sys - -from app import conf -from app.main import app - - -sys.path.append(os.path.dirname(__file__)) - -if __name__ == '__main__': - print('Running server on http://%s:%d' % (conf.HOST, conf.PORT)) - app.run(debug=conf.DEBUG, host=conf.HOST, port=conf.PORT) From 723f46d5237bd3e2ca3b4785f33fbdce8ac02871 Mon Sep 17 00:00:00 2001 From: Chuck Harmston Date: Thu, 25 Feb 2016 11:10:44 -0700 Subject: [PATCH 2/4] Renames 'app' to 'recommendation'. --- Dockerfile | 3 +- app/__init__.py => __init__.py | 0 app/.dockerignore | 3 -- app/conf.py | 17 ------ app/entry.sh | 3 -- app/main.py | 5 -- app/search/classification/__init__.py | 10 ---- app/tasks/__init__.py | 1 - conf/web.sh | 4 -- {app/search => recommendation}/__init__.py | 0 {app => recommendation}/cors.py | 0 {app => recommendation}/factory.py | 8 +-- recommendation/main.py | 5 ++ {app => recommendation}/memcached.py | 2 +- {app => recommendation}/memorize.py | 2 +- .../search}/__init__.py | 0 .../search/classification/__init__.py | 10 ++++ .../search/classification/base.py | 0 .../search/classification/domain.py | 2 +- .../search/classification/embedly.py | 11 ++-- .../search/classification/tests}/__init__.py | 0 .../search/classification/tests/test_base.py | 2 +- .../classification/tests/test_domain.py | 2 +- .../classification/tests/test_embedly.py | 22 ++++---- .../classification/tests/test_wikipedia.py | 14 ++--- .../search/classification/wikipedia.py | 4 +- .../search/query}/__init__.py | 0 {app => recommendation}/search/query/base.py | 2 +- .../search/query/tests}/__init__.py | 0 .../search/query/tests/test_base.py | 15 +++--- .../search/query/tests/test_yahoo.py | 8 +-- {app => recommendation}/search/query/yahoo.py | 6 +-- .../search/recommendation.py | 6 +-- .../search/suggest}/__init__.py | 0 .../search/suggest/base.py | 2 +- .../search/suggest/bing.py | 4 +- .../search/suggest}/tests/__init__.py | 0 .../search/suggest/tests/test_base.py | 8 +-- .../search/suggest/tests/test_bing.py | 6 +-- .../search}/tests/__init__.py | 0 .../search/tests/test_recommendation.py | 52 +++++++++++-------- recommendation/tasks/__init__.py | 1 + .../tasks/task_recommend.py | 6 +-- .../tasks}/tests/__init__.py | 0 .../tasks/tests/test_recommend.py | 9 ++-- .../tests}/__init__.py | 0 {app => recommendation}/tests/memcached.py | 4 +- {app => recommendation}/tests/test_cors.py | 4 +- {app => recommendation}/tests/test_factory.py | 8 +-- {app => recommendation}/tests/test_main.py | 20 +++---- .../tests/test_memorize.py | 6 +-- {app => recommendation}/tests/util.py | 2 +- .../views}/__init__.py | 0 {app => recommendation}/views/main.py | 6 +-- {app => recommendation}/views/status.py | 6 +-- recommendation/views/tests/__init__.py | 0 .../views/tests/test_main.py | 20 +++---- .../views/tests/test_status.py | 47 +++++++++-------- setup.cfg | 2 +- 59 files changed, 184 insertions(+), 196 deletions(-) rename app/__init__.py => __init__.py (100%) delete mode 100644 app/.dockerignore delete mode 100644 app/conf.py delete mode 100755 app/entry.sh delete mode 100644 app/main.py delete mode 100644 app/search/classification/__init__.py delete mode 100644 app/tasks/__init__.py rename {app/search => recommendation}/__init__.py (100%) rename {app => recommendation}/cors.py (100%) rename {app => recommendation}/factory.py (82%) create mode 100644 recommendation/main.py rename {app => recommendation}/memcached.py (69%) rename {app => recommendation}/memorize.py (98%) rename {app/search/classification/tests => recommendation/search}/__init__.py (100%) create mode 100644 recommendation/search/classification/__init__.py rename {app => recommendation}/search/classification/base.py (100%) rename {app => recommendation}/search/classification/domain.py (88%) rename {app => recommendation}/search/classification/embedly.py (89%) rename {app/search/query => recommendation/search/classification/tests}/__init__.py (100%) rename {app => recommendation}/search/classification/tests/test_base.py (85%) rename {app => recommendation}/search/classification/tests/test_domain.py (94%) rename {app => recommendation}/search/classification/tests/test_embedly.py (86%) rename {app => recommendation}/search/classification/tests/test_wikipedia.py (87%) rename {app => recommendation}/search/classification/wikipedia.py (94%) rename {app/search/query/tests => recommendation/search/query}/__init__.py (100%) rename {app => recommendation}/search/query/base.py (98%) rename {app/search/suggest => recommendation/search/query/tests}/__init__.py (100%) rename {app => recommendation}/search/query/tests/test_base.py (84%) rename {app => recommendation}/search/query/tests/test_yahoo.py (91%) rename {app => recommendation}/search/query/yahoo.py (88%) rename {app => recommendation}/search/recommendation.py (93%) rename {app/search/suggest/tests => recommendation/search/suggest}/__init__.py (100%) rename {app => recommendation}/search/suggest/base.py (94%) rename {app => recommendation}/search/suggest/bing.py (84%) rename {app/search => recommendation/search/suggest}/tests/__init__.py (100%) rename {app => recommendation}/search/suggest/tests/test_base.py (77%) rename {app => recommendation}/search/suggest/tests/test_bing.py (82%) rename {app/tasks => recommendation/search}/tests/__init__.py (100%) rename {app => recommendation}/search/tests/test_recommendation.py (60%) create mode 100644 recommendation/tasks/__init__.py rename {app => recommendation}/tasks/task_recommend.py (55%) rename {app => recommendation/tasks}/tests/__init__.py (100%) rename {app => recommendation}/tasks/tests/test_recommend.py (58%) rename {app/views => recommendation/tests}/__init__.py (100%) rename {app => recommendation}/tests/memcached.py (83%) rename {app => recommendation}/tests/test_cors.py (84%) rename {app => recommendation}/tests/test_factory.py (81%) rename {app => recommendation}/tests/test_main.py (78%) rename {app => recommendation}/tests/test_memorize.py (94%) rename {app => recommendation}/tests/util.py (85%) rename {app/views/tests => recommendation/views}/__init__.py (100%) rename {app => recommendation}/views/main.py (82%) rename {app => recommendation}/views/status.py (92%) create mode 100644 recommendation/views/tests/__init__.py rename {app => recommendation}/views/tests/test_main.py (76%) rename {app => recommendation}/views/tests/test_status.py (71%) diff --git a/Dockerfile b/Dockerfile index cce107a..bf806f3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,4 +15,5 @@ COPY . /app ENV PYTHONPATH $PYTHONPATH:/app USER app -ENTRYPOINT /app/conf/web.sh +EXPOSE 8000 +ENTRYPOINT ["/app/conf/web.sh"] diff --git a/app/__init__.py b/__init__.py similarity index 100% rename from app/__init__.py rename to __init__.py diff --git a/app/.dockerignore b/app/.dockerignore deleted file mode 100644 index 0920af2..0000000 --- a/app/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -__pycache__ -src -.DS_Store diff --git a/app/conf.py b/app/conf.py deleted file mode 100644 index 212f2c7..0000000 --- a/app/conf.py +++ /dev/null @@ -1,17 +0,0 @@ -from os import environ as env - - -CELERY_BROKER_URL = env.get('CELERY_BROKER_URL', 'redis://redis:6379/0') -DEBUG = env.get('RECOMMENDATION_ENV', 'development') == 'development' -KEY_PREFIX = env.get('RECOMMENDATION_KEY_PREFIX', 'query_') -MEMCACHED_HOST = env.get('MEMCACHED_HOST', 'memcached:11211') - -EMBEDLY_API_KEY = env.get('EMBEDLY_API_KEY', '') - -# Yahoo BOSS API credentials. -YAHOO_OAUTH_KEY = env.get('YAHOO_OAUTH_KEY', '') -YAHOO_OAUTH_SECRET = env.get('YAHOO_OAUTH_SECRET', '') - -# For non-Docker development server. -HOST = env.get('RECOMMENDATION_HOST', '0.0.0.0') -PORT = env.get('RECOMMENDATION_PORT', 5000) diff --git a/app/entry.sh b/app/entry.sh deleted file mode 100755 index e8379ad..0000000 --- a/app/entry.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -uwsgi --http :${PORT:-8000} --wsgi-file /app/main.py --master diff --git a/app/main.py b/app/main.py deleted file mode 100644 index 8c36906..0000000 --- a/app/main.py +++ /dev/null @@ -1,5 +0,0 @@ -from app import tasks # noqa -from app.factory import create_app, create_queue - -application = create_app() -celery = create_queue() diff --git a/app/search/classification/__init__.py b/app/search/classification/__init__.py deleted file mode 100644 index e8a09cb..0000000 --- a/app/search/classification/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -from app.search.classification.domain import DomainClassifier -from app.search.classification.embedly import EmbedlyClassifier -from app.search.classification.wikipedia import WikipediaClassifier - - -CLASSIFIERS = [ - DomainClassifier, - EmbedlyClassifier, - WikipediaClassifier, -] diff --git a/app/tasks/__init__.py b/app/tasks/__init__.py deleted file mode 100644 index 9d5ccd6..0000000 --- a/app/tasks/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from app.tasks.task_recommend import recommend # noqa diff --git a/conf/web.sh b/conf/web.sh index fc61b25..9d5d9ca 100755 --- a/conf/web.sh +++ b/conf/web.sh @@ -1,7 +1,3 @@ #!/usr/bin/env bash -<<<<<<< HEAD -uwsgi --http :${PORT:-8000} --wsgi-file ../app/main.py --master -======= uwsgi --http :${PORT:-8000} --wsgi-file /app/recommendation/wsgi.py --master ->>>>>>> 579c385... fixup! WIP diff --git a/app/search/__init__.py b/recommendation/__init__.py similarity index 100% rename from app/search/__init__.py rename to recommendation/__init__.py diff --git a/app/cors.py b/recommendation/cors.py similarity index 100% rename from app/cors.py rename to recommendation/cors.py diff --git a/app/factory.py b/recommendation/factory.py similarity index 82% rename from app/factory.py rename to recommendation/factory.py index ad3ccda..a1f746d 100644 --- a/app/factory.py +++ b/recommendation/factory.py @@ -1,10 +1,10 @@ from celery import Celery from flask import Flask -from app import conf -from app.cors import cors_headers -from app.views.main import main -from app.views.status import status +from recommendation import conf +from recommendation.cors import cors_headers +from recommendation.views.main import main +from recommendation.views.status import status def create_app(): diff --git a/recommendation/main.py b/recommendation/main.py new file mode 100644 index 0000000..adeea5e --- /dev/null +++ b/recommendation/main.py @@ -0,0 +1,5 @@ +from recommendation import tasks # noqa +from recommendation.factory import create_app, create_queue + +application = create_app() +celery = create_queue() diff --git a/app/memcached.py b/recommendation/memcached.py similarity index 69% rename from app/memcached.py rename to recommendation/memcached.py index 8d07b76..a0925dd 100644 --- a/app/memcached.py +++ b/recommendation/memcached.py @@ -1,6 +1,6 @@ from memcache import Client -from app import conf +from recommendation import conf memcached = Client([conf.MEMCACHED_HOST]) diff --git a/app/memorize.py b/recommendation/memorize.py similarity index 98% rename from app/memorize.py rename to recommendation/memorize.py index e789e38..36285c4 100644 --- a/app/memorize.py +++ b/recommendation/memorize.py @@ -5,7 +5,7 @@ from inspect import getargspec from wrapt import ObjectProxy -from app.memcached import memcached +from recommendation.memcached import memcached class MemorizedObject(ObjectProxy): diff --git a/app/search/classification/tests/__init__.py b/recommendation/search/__init__.py similarity index 100% rename from app/search/classification/tests/__init__.py rename to recommendation/search/__init__.py diff --git a/recommendation/search/classification/__init__.py b/recommendation/search/classification/__init__.py new file mode 100644 index 0000000..77085dd --- /dev/null +++ b/recommendation/search/classification/__init__.py @@ -0,0 +1,10 @@ +from recommendation.search.classification.domain import DomainClassifier +from recommendation.search.classification.embedly import EmbedlyClassifier +from recommendation.search.classification.wikipedia import WikipediaClassifier + + +CLASSIFIERS = [ + DomainClassifier, + EmbedlyClassifier, + WikipediaClassifier, +] diff --git a/app/search/classification/base.py b/recommendation/search/classification/base.py similarity index 100% rename from app/search/classification/base.py rename to recommendation/search/classification/base.py diff --git a/app/search/classification/domain.py b/recommendation/search/classification/domain.py similarity index 88% rename from app/search/classification/domain.py rename to recommendation/search/classification/domain.py index 1d835e4..2389b1b 100644 --- a/app/search/classification/domain.py +++ b/recommendation/search/classification/domain.py @@ -1,4 +1,4 @@ -from app.search.classification.base import BaseClassifier +from recommendation.search.classification.base import BaseClassifier class DomainClassifier(BaseClassifier): diff --git a/app/search/classification/embedly.py b/recommendation/search/classification/embedly.py similarity index 89% rename from app/search/classification/embedly.py rename to recommendation/search/classification/embedly.py index fd68b88..27b528f 100644 --- a/app/search/classification/embedly.py +++ b/recommendation/search/classification/embedly.py @@ -2,11 +2,10 @@ from urllib.parse import urlencode import requests -import app.conf -from app.memorize import memorize - -from app.search.classification.base import BaseClassifier -from app.search.classification.wikipedia import WikipediaClassifier +from recommendation import conf +from recommendation.memorize import memorize +from recommendation.search.classification.base import BaseClassifier +from recommendation.search.classification.wikipedia import WikipediaClassifier class EmbedlyClassifier(BaseClassifier): @@ -51,7 +50,7 @@ class EmbedlyClassifier(BaseClassifier): def _api_url(self, url): return '%s?%s' % (self.api_url, urlencode({ - 'key': app.conf.EMBEDLY_API_KEY, + 'key': conf.EMBEDLY_API_KEY, 'words': 20, 'secure': 'true', 'url': url diff --git a/app/search/query/__init__.py b/recommendation/search/classification/tests/__init__.py similarity index 100% rename from app/search/query/__init__.py rename to recommendation/search/classification/tests/__init__.py diff --git a/app/search/classification/tests/test_base.py b/recommendation/search/classification/tests/test_base.py similarity index 85% rename from app/search/classification/tests/test_base.py rename to recommendation/search/classification/tests/test_base.py index ab2744f..ba17a46 100644 --- a/app/search/classification/tests/test_base.py +++ b/recommendation/search/classification/tests/test_base.py @@ -1,6 +1,6 @@ from unittest import TestCase -from app.search.classification.base import BaseClassifier +from recommendation.search.classification.base import BaseClassifier class TestBaseClassifier(TestCase): diff --git a/app/search/classification/tests/test_domain.py b/recommendation/search/classification/tests/test_domain.py similarity index 94% rename from app/search/classification/tests/test_domain.py rename to recommendation/search/classification/tests/test_domain.py index ebb300b..a7e9d2e 100644 --- a/app/search/classification/tests/test_domain.py +++ b/recommendation/search/classification/tests/test_domain.py @@ -3,7 +3,7 @@ from urllib.parse import ParseResult from nose.tools import eq_, ok_ -from app.search.classification.domain import DomainClassifier +from recommendation.search.classification.domain import DomainClassifier DOMAIN = 'www.mozilla.com' diff --git a/app/search/classification/tests/test_embedly.py b/recommendation/search/classification/tests/test_embedly.py similarity index 86% rename from app/search/classification/tests/test_embedly.py rename to recommendation/search/classification/tests/test_embedly.py index c2ed9c3..650ab0a 100644 --- a/app/search/classification/tests/test_embedly.py +++ b/recommendation/search/classification/tests/test_embedly.py @@ -5,8 +5,8 @@ from urllib.parse import parse_qs, urlparse import responses from nose.tools import eq_, ok_ -from app.search.classification.embedly import EmbedlyClassifier -from app.tests.memcached import mock_memcached +from recommendation.search.classification.embedly import EmbedlyClassifier +from recommendation.tests.memcached import mock_memcached MOCK_API_KEY = '0123456789abcdef' @@ -92,7 +92,7 @@ class TestEmbedlyClassifier(TestCase): eq_(self._matches('https://en.wikipedia.org/wiki/Mozilla'), True) eq_(self._matches('https://en.wikipedia.org/wiki/Mozilla/'), True) - @patch('app.conf.EMBEDLY_API_KEY', MOCK_API_KEY) + @patch('recommendation.conf.EMBEDLY_API_KEY', MOCK_API_KEY) def test_api_url(self): api_url = self._api_url(MOCK_RESULT_URL) api_qs = parse_qs(urlparse(api_url).query) @@ -101,8 +101,9 @@ class TestEmbedlyClassifier(TestCase): eq_(api_qs['url'][0], MOCK_RESULT_URL) eq_(api_qs['secure'][0], 'true') - @patch('app.memorize.memcached', mock_memcached) - @patch('app.search.classification.embedly.EmbedlyClassifier._api_url') + @patch('recommendation.memorize.memcached', mock_memcached) + @patch('recommendation.search.classification.embedly.EmbedlyClassifier' + '._api_url') @responses.activate def test_api_response(self, mock_api_url): mock_api_url.return_value = MOCK_API_URL @@ -116,8 +117,9 @@ class TestEmbedlyClassifier(TestCase): eq_(response_cold.cache_key, response_warm.cache_key) eq_(response_cold, response_warm, MOCK_RESPONSE) - @patch('app.memorize.memcached', mock_memcached) - @patch('app.search.classification.embedly.EmbedlyClassifier._api_url') + @patch('recommendation.memorize.memcached', mock_memcached) + @patch('recommendation.search.classification.embedly.EmbedlyClassifier' + '._api_url') @responses.activate def test_enhance(self, mock_api_url): mock_api_url.return_value = MOCK_API_URL @@ -130,8 +132,10 @@ class TestEmbedlyClassifier(TestCase): eq_(enhanced['favicon']['colors'], MOCK_RESPONSE['favicon_colors']) eq_(enhanced['favicon']['url'], MOCK_RESPONSE['favicon_url']) - @patch('app.search.classification.embedly.EmbedlyClassifier._api_response') - @patch('app.search.classification.embedly.EmbedlyClassifier._get_image') + @patch('recommendation.search.classification.embedly.EmbedlyClassifier' + '._api_response') + @patch('recommendation.search.classification.embedly.EmbedlyClassifier' + '._get_image') def test_enhance_no_images(self, mock_get_image, mock_api_response): mock_api_response.return_value = MOCK_RESPONSE mock_get_image.side_effect = KeyError diff --git a/app/search/classification/tests/test_wikipedia.py b/recommendation/search/classification/tests/test_wikipedia.py similarity index 87% rename from app/search/classification/tests/test_wikipedia.py rename to recommendation/search/classification/tests/test_wikipedia.py index f97cf67..22f260e 100644 --- a/app/search/classification/tests/test_wikipedia.py +++ b/recommendation/search/classification/tests/test_wikipedia.py @@ -5,8 +5,8 @@ from urllib.parse import parse_qs, urlparse import responses from nose.tools import eq_, ok_ -from app.search.classification.wikipedia import WikipediaClassifier -from app.tests.memcached import mock_memcached +from recommendation.search.classification.wikipedia import WikipediaClassifier +from recommendation.tests.memcached import mock_memcached SLUG = 'Mozilla' @@ -70,8 +70,9 @@ class TestWikipediaClassifier(TestCase): actual[4] = parse_qs(actual[4]) eq_(expected, actual) - @patch('app.memorize.memcached', mock_memcached) - @patch('app.search.classification.wikipedia.WikipediaClassifier._api_url') + @patch('recommendation.memorize.memcached', mock_memcached) + @patch('recommendation.search.classification.wikipedia.WikipediaClassifier' + '._api_url') @responses.activate def test_api_response(self, mock_api_url): """ @@ -94,8 +95,9 @@ class TestWikipediaClassifier(TestCase): eq_(response_cold.cache_key, response_warm.cache_key) eq_(response_cold, response_warm, EXPECTED_SANITIZED_RESPONSE) - @patch('app.memorize.memcached', mock_memcached) - @patch('app.search.classification.wikipedia.WikipediaClassifier._api_url') + @patch('recommendation.memorize.memcached', mock_memcached) + @patch('recommendation.search.classification.wikipedia.WikipediaClassifier' + '._api_url') @responses.activate def test_enhance(self, mock_api_url): """ diff --git a/app/search/classification/wikipedia.py b/recommendation/search/classification/wikipedia.py similarity index 94% rename from app/search/classification/wikipedia.py rename to recommendation/search/classification/wikipedia.py index 677580c..1f7c499 100644 --- a/app/search/classification/wikipedia.py +++ b/recommendation/search/classification/wikipedia.py @@ -2,8 +2,8 @@ from urllib.parse import urlencode, urlunparse import requests -from app.memorize import memorize -from app.search.classification.base import BaseClassifier +from recommendation.memorize import memorize +from recommendation.search.classification.base import BaseClassifier class WikipediaClassifier(BaseClassifier): diff --git a/app/search/query/tests/__init__.py b/recommendation/search/query/__init__.py similarity index 100% rename from app/search/query/tests/__init__.py rename to recommendation/search/query/__init__.py diff --git a/app/search/query/base.py b/recommendation/search/query/base.py similarity index 98% rename from app/search/query/base.py rename to recommendation/search/query/base.py index f65361f..c061f8a 100644 --- a/app/search/query/base.py +++ b/recommendation/search/query/base.py @@ -1,6 +1,6 @@ import bleach -from app.memorize import memorize +from recommendation.memorize import memorize class BaseQueryEngine(object): diff --git a/app/search/suggest/__init__.py b/recommendation/search/query/tests/__init__.py similarity index 100% rename from app/search/suggest/__init__.py rename to recommendation/search/query/tests/__init__.py diff --git a/app/search/query/tests/test_base.py b/recommendation/search/query/tests/test_base.py similarity index 84% rename from app/search/query/tests/test_base.py rename to recommendation/search/query/tests/test_base.py index fb89369..57d571b 100644 --- a/app/search/query/tests/test_base.py +++ b/recommendation/search/query/tests/test_base.py @@ -3,8 +3,8 @@ from unittest.mock import patch from nose.tools import eq_, ok_ -from app.search.query.base import BaseQueryEngine -from app.tests.memcached import mock_memcached +from recommendation.search.query.base import BaseQueryEngine +from recommendation.tests.memcached import mock_memcached QUERY = 'loveland' @@ -41,9 +41,10 @@ class TestBaseQueryEngine(TestCase): def test_get_best_result(self): eq_(self.instance.get_best_result(MOCK_RESULTS), MOCK_RESULTS[0]) - @patch('app.search.query.base.BaseQueryEngine.get_result_abstract') - @patch('app.search.query.base.BaseQueryEngine.get_result_title') - @patch('app.search.query.base.BaseQueryEngine.get_result_url') + @patch('recommendation.search.query.base.BaseQueryEngine' + '.get_result_abstract') + @patch('recommendation.search.query.base.BaseQueryEngine.get_result_title') + @patch('recommendation.search.query.base.BaseQueryEngine.get_result_url') def test_sanitize_result(self, *args): result = self.instance.sanitize_result(MOCK_RESULTS[0]) ok_(all(k in result.keys() for k in ['abstract', 'title', 'url'])) @@ -69,8 +70,8 @@ class TestBaseQueryEngine(TestCase): eq_(self.instance.get_result_abstract(MOCK_RESULTS[0]), MOCK_RESULT_SANITIZED['abstract']) - @patch('app.search.query.base.BaseQueryEngine.fetch') - @patch('app.memorize.memcached', mock_memcached) + @patch('recommendation.search.query.base.BaseQueryEngine.fetch') + @patch('recommendation.memorize.memcached', mock_memcached) def test_search(self, mock_fetch): mock_fetch.return_value = MOCK_RESULTS results_cold = self.instance.search(QUERY) diff --git a/app/search/query/tests/test_yahoo.py b/recommendation/search/query/tests/test_yahoo.py similarity index 91% rename from app/search/query/tests/test_yahoo.py rename to recommendation/search/query/tests/test_yahoo.py index 6f26664..a2e0198 100644 --- a/app/search/query/tests/test_yahoo.py +++ b/recommendation/search/query/tests/test_yahoo.py @@ -5,8 +5,8 @@ from urllib.parse import parse_qs, urlparse import responses from nose.tools import eq_, ok_ -from app.search.query.yahoo import YahooQueryEngine -from app.tests.memcached import mock_memcached +from recommendation.search.query.yahoo import YahooQueryEngine +from recommendation.tests.memcached import mock_memcached QUERY = 'loveland' @@ -56,7 +56,7 @@ class TestYahooClassifier(TestCase): ok_(len(query_string['oauth_version'])) eq_(query_string['q'][0], QUERY) - @patch('app.memorize.memcached', mock_memcached) + @patch('recommendation.memorize.memcached', mock_memcached) @responses.activate def test_fetch(self): responses.add(responses.GET, YahooQueryEngine.url, @@ -68,7 +68,7 @@ class TestYahooClassifier(TestCase): eq_(response_cold.cache_key, response_warm.cache_key) eq_(response_cold, response_warm, MOCK_RESPONSE) - @patch('app.memorize.memcached', mock_memcached) + @patch('recommendation.memorize.memcached', mock_memcached) @responses.activate def test_sanitize_response(self): responses.add(responses.GET, YahooQueryEngine.url, diff --git a/app/search/query/yahoo.py b/recommendation/search/query/yahoo.py similarity index 88% rename from app/search/query/yahoo.py rename to recommendation/search/query/yahoo.py index c57ff67..b78bbfc 100644 --- a/app/search/query/yahoo.py +++ b/recommendation/search/query/yahoo.py @@ -4,9 +4,9 @@ from urllib.parse import quote_plus import oauth2 import requests -from app import conf -from app.memorize import memorize -from app.search.query.base import BaseQueryEngine +from recommendation import conf +from recommendation.memorize import memorize +from recommendation.search.query.base import BaseQueryEngine class YahooQueryEngine(BaseQueryEngine): diff --git a/app/search/recommendation.py b/recommendation/search/recommendation.py similarity index 93% rename from app/search/recommendation.py rename to recommendation/search/recommendation.py index b9e1820..e0ade95 100644 --- a/app/search/recommendation.py +++ b/recommendation/search/recommendation.py @@ -1,6 +1,6 @@ -from app.search.classification import CLASSIFIERS -from app.search.query.yahoo import YahooQueryEngine -from app.search.suggest.bing import BingSuggestionEngine +from recommendation.search.classification import CLASSIFIERS +from recommendation.search.query.yahoo import YahooQueryEngine +from recommendation.search.suggest.bing import BingSuggestionEngine class SearchRecommendation(object): diff --git a/app/search/suggest/tests/__init__.py b/recommendation/search/suggest/__init__.py similarity index 100% rename from app/search/suggest/tests/__init__.py rename to recommendation/search/suggest/__init__.py diff --git a/app/search/suggest/base.py b/recommendation/search/suggest/base.py similarity index 94% rename from app/search/suggest/base.py rename to recommendation/search/suggest/base.py index eb94c4c..5552424 100644 --- a/app/search/suggest/base.py +++ b/recommendation/search/suggest/base.py @@ -1,4 +1,4 @@ -from app.memorize import memorize +from recommendation.memorize import memorize class BaseSuggestionEngine(object): diff --git a/app/search/suggest/bing.py b/recommendation/search/suggest/bing.py similarity index 84% rename from app/search/suggest/bing.py rename to recommendation/search/suggest/bing.py index cc5d6de..049da67 100644 --- a/app/search/suggest/bing.py +++ b/recommendation/search/suggest/bing.py @@ -1,7 +1,7 @@ import requests -from app.memorize import memorize -from app.search.suggest.base import BaseSuggestionEngine +from recommendation.memorize import memorize +from recommendation.search.suggest.base import BaseSuggestionEngine class BingSuggestionEngine(BaseSuggestionEngine): diff --git a/app/search/tests/__init__.py b/recommendation/search/suggest/tests/__init__.py similarity index 100% rename from app/search/tests/__init__.py rename to recommendation/search/suggest/tests/__init__.py diff --git a/app/search/suggest/tests/test_base.py b/recommendation/search/suggest/tests/test_base.py similarity index 77% rename from app/search/suggest/tests/test_base.py rename to recommendation/search/suggest/tests/test_base.py index 3d64aa6..e03c46d 100644 --- a/app/search/suggest/tests/test_base.py +++ b/recommendation/search/suggest/tests/test_base.py @@ -3,8 +3,8 @@ from unittest import TestCase from mock import patch from nose.tools import eq_, ok_ -from app.search.suggest.base import BaseSuggestionEngine -from app.tests.memcached import mock_memcached +from recommendation.search.suggest.base import BaseSuggestionEngine +from recommendation.tests.memcached import mock_memcached QUERY = '' @@ -29,8 +29,8 @@ class TestBaseSuggestionEngine(TestCase): def test_sanitize(self): eq_(self.instance.sanitize(RESULTS), RESULTS) - @patch('app.search.suggest.base.BaseSuggestionEngine.fetch') - @patch('app.memorize.memcached', mock_memcached) + @patch('recommendation.search.suggest.base.BaseSuggestionEngine.fetch') + @patch('recommendation.memorize.memcached', mock_memcached) def test_search(self, mock_fetch): mock_fetch.return_value = RESULTS results_cold = self.instance.search(QUERY) diff --git a/app/search/suggest/tests/test_bing.py b/recommendation/search/suggest/tests/test_bing.py similarity index 82% rename from app/search/suggest/tests/test_bing.py rename to recommendation/search/suggest/tests/test_bing.py index fd5d8c2..097154e 100644 --- a/app/search/suggest/tests/test_bing.py +++ b/recommendation/search/suggest/tests/test_bing.py @@ -4,8 +4,8 @@ import responses from mock import patch from nose.tools import eq_, ok_ -from app.search.suggest.bing import BingSuggestionEngine -from app.tests.memcached import mock_memcached +from recommendation.search.suggest.bing import BingSuggestionEngine +from recommendation.tests.memcached import mock_memcached QUERY = 'original query' @@ -23,7 +23,7 @@ class TestBingSuggestionEngine(TestCase): def setUp(self): self.instance = BingSuggestionEngine(QUERY) - @patch('app.memorize.memcached', mock_memcached) + @patch('recommendation.memorize.memcached', mock_memcached) @responses.activate def test_fetch(self): responses.add(responses.GET, BingSuggestionEngine.url, json=RESPONSE, diff --git a/app/tasks/tests/__init__.py b/recommendation/search/tests/__init__.py similarity index 100% rename from app/tasks/tests/__init__.py rename to recommendation/search/tests/__init__.py diff --git a/app/search/tests/test_recommendation.py b/recommendation/search/tests/test_recommendation.py similarity index 60% rename from app/search/tests/test_recommendation.py rename to recommendation/search/tests/test_recommendation.py index 273a859..a59d728 100644 --- a/app/search/tests/test_recommendation.py +++ b/recommendation/search/tests/test_recommendation.py @@ -3,18 +3,18 @@ from unittest import TestCase from mock import patch from nose.tools import eq_, ok_ -from app.search.classification.domain import DomainClassifier -from app.search.classification.tests.test_domain import DOMAIN -from app.search.recommendation import SearchRecommendation -from app.search.suggest.base import BaseSuggestionEngine -from app.search.suggest.bing import BingSuggestionEngine -from app.search.suggest.tests.test_bing import (QUERY as BING_QUERY, - RESULTS as BING_RESULTS) -from app.search.query.base import BaseQueryEngine -from app.search.query.yahoo import YahooQueryEngine -from app.search.query.tests.test_yahoo import (QUERY as YAHOO_QUERY, - MOCK_RESPONSE as YAHOO_RESPONSE) -from app.tests.memcached import mock_memcached +from recommendation.search.classification.domain import DomainClassifier +from recommendation.search.classification.tests.test_domain import DOMAIN +from recommendation.search.recommendation import SearchRecommendation +from recommendation.search.suggest.base import BaseSuggestionEngine +from recommendation.search.suggest.bing import BingSuggestionEngine +from recommendation.search.suggest.tests.test_bing import ( + QUERY as BING_QUERY, RESULTS as BING_RESULTS) +from recommendation.search.query.base import BaseQueryEngine +from recommendation.search.query.yahoo import YahooQueryEngine +from recommendation.search.query.tests.test_yahoo import ( + QUERY as YAHOO_QUERY, MOCK_RESPONSE as YAHOO_RESPONSE) +from recommendation.tests.memcached import mock_memcached QUERY = 'Cubs' @@ -35,7 +35,8 @@ class TestSearchRecommendation(TestCase): engine = self.instance.get_query_engine() ok_(issubclass(engine, BaseQueryEngine)) - @patch('app.search.classification.domain.DomainClassifier.is_match') + @patch('recommendation.search.classification.domain.DomainClassifier' + '.is_match') def test_get_classifiers(self, mock_match): mock_match.return_value = True classifiers = self.instance.get_classifiers({ @@ -45,9 +46,9 @@ class TestSearchRecommendation(TestCase): ok_(isinstance(classifiers[0], DomainClassifier)) return classifiers - @patch(('app.search.recommendation.SearchRecommendation.get_suggestion_eng' - 'ine')) - @patch('app.search.suggest.bing.BingSuggestionEngine.search') + @patch(('recommendation.search.recommendation.SearchRecommendation' + '.get_suggestion_engine')) + @patch('recommendation.search.suggest.bing.BingSuggestionEngine.search') def test_get_suggestions(self, mock_bing, mock_suggestion_engine): mock_bing.return_value = BING_RESULTS mock_suggestion_engine.return_value = BingSuggestionEngine @@ -57,8 +58,9 @@ class TestSearchRecommendation(TestCase): def test_get_top_suggestion(self): eq_(self.instance.get_top_suggestion(BING_RESULTS), BING_RESULTS[0]) - @patch('app.search.recommendation.SearchRecommendation.get_query_engine') - @patch('app.search.query.yahoo.YahooQueryEngine.search') + @patch('recommendation.search.recommendation.SearchRecommendation' + '.get_query_engine') + @patch('recommendation.search.query.yahoo.YahooQueryEngine.search') def test_do_query(self, mock_yahoo, mock_query_engine): response = YAHOO_RESPONSE['bossresponse']['web']['results'][0] mock_yahoo.return_value = response @@ -66,11 +68,15 @@ class TestSearchRecommendation(TestCase): eq_(self.instance.do_query(YAHOO_QUERY), response) eq_(mock_yahoo.call_count, 1) - @patch('app.search.recommendation.SearchRecommendation.get_classifiers') - @patch('app.search.recommendation.SearchRecommendation.do_query') - @patch('app.search.recommendation.SearchRecommendation.get_top_suggestion') - @patch('app.search.recommendation.SearchRecommendation.get_suggestions') - @patch('app.memorize.memcached', mock_memcached) + @patch('recommendation.search.recommendation.SearchRecommendation' + '.get_classifiers') + @patch('recommendation.search.recommendation.SearchRecommendation' + '.do_query') + @patch('recommendation.search.recommendation.SearchRecommendation' + '.get_top_suggestion') + @patch('recommendation.search.recommendation.SearchRecommendation' + '.get_suggestions') + @patch('recommendation.memorize.memcached', mock_memcached) def test_do_search_get_recommendation(self, mock_suggestions, mock_top_suggestion, mock_result, mock_classifiers): diff --git a/recommendation/tasks/__init__.py b/recommendation/tasks/__init__.py new file mode 100644 index 0000000..cd735d9 --- /dev/null +++ b/recommendation/tasks/__init__.py @@ -0,0 +1 @@ +from recommendation.tasks.task_recommend import recommend # noqa diff --git a/app/tasks/task_recommend.py b/recommendation/tasks/task_recommend.py similarity index 55% rename from app/tasks/task_recommend.py rename to recommendation/tasks/task_recommend.py index ad27e9d..545954d 100644 --- a/app/tasks/task_recommend.py +++ b/recommendation/tasks/task_recommend.py @@ -1,6 +1,6 @@ -from app.factory import create_queue -from app.memcached import memcached -from app.search.recommendation import SearchRecommendation +from recommendation.factory import create_queue +from recommendation.memcached import memcached +from recommendation.search.recommendation import SearchRecommendation queue = create_queue() diff --git a/app/tests/__init__.py b/recommendation/tasks/tests/__init__.py similarity index 100% rename from app/tests/__init__.py rename to recommendation/tasks/tests/__init__.py diff --git a/app/tasks/tests/test_recommend.py b/recommendation/tasks/tests/test_recommend.py similarity index 58% rename from app/tasks/tests/test_recommend.py rename to recommendation/tasks/tests/test_recommend.py index e76499a..8e74485 100644 --- a/app/tasks/tests/test_recommend.py +++ b/recommendation/tasks/tests/test_recommend.py @@ -3,7 +3,7 @@ from unittest import TestCase from mock import patch from nose.tools import eq_ -from app.tests.memcached import mock_memcached +from recommendation.tests.memcached import mock_memcached KEY = 'query_key' @@ -13,12 +13,13 @@ RESULTS = { } -@patch('app.tasks.task_recommend.memcached', mock_memcached) +@patch('recommendation.tasks.task_recommend.memcached', mock_memcached) class TestRecommendTask(TestCase): - @patch('app.search.recommendation.SearchRecommendation.do_search') + @patch('recommendation.search.recommendation.SearchRecommendation' + '.do_search') def test_recommend(self, mock_do_search): - from app.tasks.task_recommend import recommend + from recommendation.tasks.task_recommend import recommend mock_do_search.return_value = RESULTS results = recommend.apply(args=[QUERY, KEY]).get() eq_(results, RESULTS) diff --git a/app/views/__init__.py b/recommendation/tests/__init__.py similarity index 100% rename from app/views/__init__.py rename to recommendation/tests/__init__.py diff --git a/app/tests/memcached.py b/recommendation/tests/memcached.py similarity index 83% rename from app/tests/memcached.py rename to recommendation/tests/memcached.py index 83fb0b9..e64b20e 100644 --- a/app/tests/memcached.py +++ b/recommendation/tests/memcached.py @@ -5,12 +5,12 @@ class FakeMemcached(object): `memorize` decorator like so: from unittest.mock import patch - from app.tests.memcached import mock_memcached + from recommendation.tests.memcached import mock_memcached def tearDown(self): mock_memcached.flush_all() - @patch('app.memorize.memcached', mock_memcached) + @patch('recommendation.memorize.memcached', mock_memcached) def test_foo(self): assert(True) """ diff --git a/app/tests/test_cors.py b/recommendation/tests/test_cors.py similarity index 84% rename from app/tests/test_cors.py rename to recommendation/tests/test_cors.py index 7c8b999..9bd6b1d 100644 --- a/app/tests/test_cors.py +++ b/recommendation/tests/test_cors.py @@ -1,7 +1,7 @@ from nose.tools import ok_ -from app.cors import cors_headers -from app.tests.util import AppTestCase +from recommendation.cors import cors_headers +from recommendation.tests.util import AppTestCase class TestCORS(AppTestCase): diff --git a/app/tests/test_factory.py b/recommendation/tests/test_factory.py similarity index 81% rename from app/tests/test_factory.py rename to recommendation/tests/test_factory.py index 53266fb..18d582b 100644 --- a/app/tests/test_factory.py +++ b/recommendation/tests/test_factory.py @@ -4,16 +4,16 @@ from celery.app.base import Celery from mock import patch from nose.tools import eq_, ok_ -from app.cors import cors_headers -from app.factory import create_app, create_queue -from app.tests.util import AppTestCase +from recommendation.cors import cors_headers +from recommendation.factory import create_app, create_queue +from recommendation.tests.util import AppTestCase BROKER_URL = 'http://celery.carrots/broccoli' class TestCreateApp(TestCase): - @patch('app.factory.conf.CELERY_BROKER_URL', BROKER_URL) + @patch('recommendation.factory.conf.CELERY_BROKER_URL', BROKER_URL) def test_create_app(self): app = create_app() ok_(cors_headers in app.after_request_funcs[None]) diff --git a/app/tests/test_main.py b/recommendation/tests/test_main.py similarity index 78% rename from app/tests/test_main.py rename to recommendation/tests/test_main.py index 18887a1..07729cf 100644 --- a/app/tests/test_main.py +++ b/recommendation/tests/test_main.py @@ -4,8 +4,8 @@ from flask.ext.testing import TestCase as FlaskTestCase from mock import patch from nose.tools import eq_, ok_ -from app.cors import cors_headers -from app.tests.memcached import mock_memcached +from recommendation.cors import cors_headers +from recommendation.tests.memcached import mock_memcached KEY = 'query_key' @@ -17,7 +17,7 @@ EXCEPTION_ARGS = ['args'] EXCEPTION = RuntimeError(*EXCEPTION_ARGS) -@patch('app.tasks.task_recommend.memcached', mock_memcached) +@patch('recommendation.tasks.task_recommend.memcached', mock_memcached) class TestApp(FlaskTestCase): debug = False @@ -25,7 +25,7 @@ class TestApp(FlaskTestCase): mock_memcached.flush_all() def create_app(self): - from app.main import application + from recommendation.main import application app = application app.config['DEBUG'] = self.debug app.config['PRESERVE_CONTEXT_ON_EXCEPTION'] = False @@ -55,21 +55,21 @@ class TestApp(FlaskTestCase): def test_no_query(self): eq_(self._get('/').status_code, 400) - @patch('app.tasks.task_recommend.memcached.get') + @patch('recommendation.tasks.task_recommend.memcached.get') def test_exception(self, mock_get): mock_get.side_effect = EXCEPTION response = self._query(QUERY) eq_(response.status_code, 500) eq_(response.json, {}) - @patch('app.tasks.task_recommend.memcached.get') + @patch('recommendation.tasks.task_recommend.memcached.get') def test_cache_hit(self, mock_get): mock_get.return_value = RESULTS eq_(self._query(QUERY).status_code, 200) eq_(self._query(QUERY).json, RESULTS) - @patch('app.tasks.task_recommend.memcached.get') - @patch('app.tasks.task_recommend.recommend.delay') + @patch('recommendation.tasks.task_recommend.memcached.get') + @patch('recommendation.tasks.task_recommend.recommend.delay') def test_cache_miss(self, mock_delay, mock_get): mock_get.return_value = None eq_(self._query(QUERY).status_code, 202) @@ -79,8 +79,8 @@ class TestApp(FlaskTestCase): class TestAppDebug(TestApp): debug = True - @patch('app.tasks.task_recommend.memcached.get') - @patch('app.tasks.task_recommend.recommend.delay') + @patch('recommendation.tasks.task_recommend.memcached.get') + @patch('recommendation.tasks.task_recommend.recommend.delay') def test_exception(self, mock_delay, mock_get): mock_get.side_effect = EXCEPTION response = self._query(QUERY) diff --git a/app/tests/test_memorize.py b/recommendation/tests/test_memorize.py similarity index 94% rename from app/tests/test_memorize.py rename to recommendation/tests/test_memorize.py index 0247dd4..f7bc321 100644 --- a/app/tests/test_memorize.py +++ b/recommendation/tests/test_memorize.py @@ -5,8 +5,8 @@ from mock import patch from nose.tools import eq_, ok_ from wrapt import ObjectProxy -from app.memorize import memorize, MemorizedObject -from app.tests.memcached import mock_memcached +from recommendation.memorize import memorize, MemorizedObject +from recommendation.tests.memcached import mock_memcached PREFIX = 'my_prefix' @@ -58,7 +58,7 @@ class TestMemorizedObject(TestCase): self.instance.cache_key = True -@patch('app.memorize.memcached', mock_memcached) +@patch('recommendation.memorize.memcached', mock_memcached) class TestMemorize(TestCase): def setUp(self): self.obj = SampleObject() diff --git a/app/tests/util.py b/recommendation/tests/util.py similarity index 85% rename from app/tests/util.py rename to recommendation/tests/util.py index 45791c9..7b95914 100644 --- a/app/tests/util.py +++ b/recommendation/tests/util.py @@ -1,6 +1,6 @@ from flask.ext.testing import TestCase as FlaskTestCase -from app.factory import create_app +from recommendation.factory import create_app class AppTestCase(FlaskTestCase): diff --git a/app/views/tests/__init__.py b/recommendation/views/__init__.py similarity index 100% rename from app/views/tests/__init__.py rename to recommendation/views/__init__.py diff --git a/app/views/main.py b/recommendation/views/main.py similarity index 82% rename from app/views/main.py rename to recommendation/views/main.py index c88703d..363e2ea 100644 --- a/app/views/main.py +++ b/recommendation/views/main.py @@ -2,8 +2,8 @@ import hashlib from flask import abort, current_app, Blueprint, jsonify, request -from app import conf -from app.memcached import memcached +from recommendation import conf +from recommendation.memcached import memcached main = Blueprint('main', __name__) @@ -11,7 +11,7 @@ main = Blueprint('main', __name__) @main.route('/') def view(): - from app.tasks.task_recommend import recommend + from recommendation.tasks.task_recommend import recommend query = request.args.get('q') if not query: abort(400) diff --git a/app/views/status.py b/recommendation/views/status.py similarity index 92% rename from app/views/status.py rename to recommendation/views/status.py index 2d94ae2..201f2ef 100644 --- a/app/views/status.py +++ b/recommendation/views/status.py @@ -2,7 +2,7 @@ from celery.app.control import Control from flask import abort, Blueprint from redis.exceptions import ConnectionError as RedisConnectionError -from app.memcached import memcached +from recommendation.memcached import memcached status = Blueprint('status', __name__) @@ -32,7 +32,7 @@ def redis_status(): Since our application should not have access to the Redis server, we test this by instantiating a Celery Control and attempting to ping it. """ - from app.factory import create_queue + from recommendation.factory import create_queue try: Control(app=create_queue()).ping(timeout=1) except RedisConnectionError: @@ -44,7 +44,7 @@ def celery_status(): Raises ServiceDown if any Celery worker servers are down, if any clusters have no workers, or if any workers are down. """ - from app.factory import create_queue + from recommendation.factory import create_queue clusters = Control(app=create_queue()).ping(timeout=1) if not clusters: raise ServiceDown() diff --git a/recommendation/views/tests/__init__.py b/recommendation/views/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/views/tests/test_main.py b/recommendation/views/tests/test_main.py similarity index 76% rename from app/views/tests/test_main.py rename to recommendation/views/tests/test_main.py index 7b2bda9..9f1b4e5 100644 --- a/app/views/tests/test_main.py +++ b/recommendation/views/tests/test_main.py @@ -3,9 +3,9 @@ from urllib.parse import urlencode from mock import patch from nose.tools import eq_, ok_ -from app.cors import cors_headers -from app.tests.memcached import mock_memcached -from app.tests.util import AppTestCase +from recommendation.cors import cors_headers +from recommendation.tests.memcached import mock_memcached +from recommendation.tests.util import AppTestCase KEY = 'query_key' @@ -17,7 +17,7 @@ EXCEPTION_ARGS = ['args'] EXCEPTION = RuntimeError(*EXCEPTION_ARGS) -@patch('app.tasks.task_recommend.memcached', mock_memcached) +@patch('recommendation.tasks.task_recommend.memcached', mock_memcached) class TestMain(AppTestCase): debug = False @@ -48,21 +48,21 @@ class TestMain(AppTestCase): def test_no_query(self): eq_(self._get('/').status_code, 400) - @patch('app.tasks.task_recommend.memcached.get') + @patch('recommendation.tasks.task_recommend.memcached.get') def test_exception(self, mock_get): mock_get.side_effect = EXCEPTION response = self._query(QUERY) eq_(response.status_code, 500) eq_(response.json, {}) - @patch('app.tasks.task_recommend.memcached.get') + @patch('recommendation.tasks.task_recommend.memcached.get') def test_cache_hit(self, mock_get): mock_get.return_value = RESULTS eq_(self._query(QUERY).status_code, 200) eq_(self._query(QUERY).json, RESULTS) - @patch('app.tasks.task_recommend.memcached.get') - @patch('app.tasks.task_recommend.recommend.delay') + @patch('recommendation.tasks.task_recommend.memcached.get') + @patch('recommendation.tasks.task_recommend.recommend.delay') def test_cache_miss(self, mock_delay, mock_get): mock_get.return_value = None eq_(self._query(QUERY).status_code, 202) @@ -72,8 +72,8 @@ class TestMain(AppTestCase): class TestMainDebug(TestMain): debug = True - @patch('app.tasks.task_recommend.memcached.get') - @patch('app.tasks.task_recommend.recommend.delay') + @patch('recommendation.tasks.task_recommend.memcached.get') + @patch('recommendation.tasks.task_recommend.recommend.delay') def test_exception(self, mock_delay, mock_get): mock_get.side_effect = EXCEPTION response = self._query(QUERY) diff --git a/app/views/tests/test_status.py b/recommendation/views/tests/test_status.py similarity index 71% rename from app/views/tests/test_status.py rename to recommendation/views/tests/test_status.py index 6e66ce7..aeb6266 100644 --- a/app/views/tests/test_status.py +++ b/recommendation/views/tests/test_status.py @@ -2,9 +2,10 @@ from mock import patch from nose.tools import eq_, ok_ from redis.exceptions import ConnectionError as RedisError -from app.views.status import (celery_status, memcached_status, redis_status, - ServiceDown) -from app.tests.util import AppTestCase +from recommendation.views.status import (celery_status, memcached_status, + redis_status, ServiceDown) +from recommendation.tests.util import AppTestCase + MEMCACHED_WORKER_BAD = {'not ok': 'not pong'} MEMCACHED_WORKER_OK = {'ok': 'pong'} @@ -32,9 +33,9 @@ class TestStatusViews(AppTestCase): eq_(response.status_code, 200) ok_(not response.data) - @patch('app.views.status.celery_status') - @patch('app.views.status.memcached_status') - @patch('app.views.status.redis_status') + @patch('recommendation.views.status.celery_status') + @patch('recommendation.views.status.memcached_status') + @patch('recommendation.views.status.redis_status') def test_heartbeat_pass(self, mock_celery, mock_memcached, mock_redis): response = self.client.get('/__heartbeat__') eq_(response.status_code, 200) @@ -42,18 +43,18 @@ class TestStatusViews(AppTestCase): eq_(mock_memcached.call_count, 1) eq_(mock_redis.call_count, 1) - @patch('app.views.status.celery_status') - @patch('app.views.status.memcached_status') - @patch('app.views.status.redis_status') + @patch('recommendation.views.status.celery_status') + @patch('recommendation.views.status.memcached_status') + @patch('recommendation.views.status.redis_status') def test_heartbeat_celery(self, mock_celery, mock_memcached, mock_redis): mock_celery.side_effect = ServiceDown response = self.client.get('/__heartbeat__') eq_(response.status_code, 500) eq_(mock_celery.call_count, 1) - @patch('app.views.status.celery_status') - @patch('app.views.status.memcached_status') - @patch('app.views.status.redis_status') + @patch('recommendation.views.status.celery_status') + @patch('recommendation.views.status.memcached_status') + @patch('recommendation.views.status.redis_status') def test_heartbeat_memcached(self, mock_celery, mock_memcached, mock_redis): mock_memcached.side_effect = ServiceDown @@ -61,57 +62,57 @@ class TestStatusViews(AppTestCase): eq_(response.status_code, 500) eq_(mock_memcached.call_count, 1) - @patch('app.views.status.celery_status') - @patch('app.views.status.memcached_status') - @patch('app.views.status.redis_status') + @patch('recommendation.views.status.celery_status') + @patch('recommendation.views.status.memcached_status') + @patch('recommendation.views.status.redis_status') def test_heartbeat_redis(self, mock_celery, mock_memcached, mock_redis): mock_redis.side_effect = ServiceDown response = self.client.get('/__heartbeat__') eq_(response.status_code, 500) eq_(mock_redis.call_count, 1) - @patch('app.views.status.memcached.set') + @patch('recommendation.views.status.memcached.set') def test_memcached_status_pass(self, mock_set): mock_set.return_value = True memcached_status() self.assert_(True) - @patch('app.views.status.memcached.set') + @patch('recommendation.views.status.memcached.set') def test_memcached_status_fail(self, mock_set): mock_set.return_value = 0 with self.assertRaises(ServiceDown): memcached_status() - @patch('app.views.status.Control.ping') + @patch('recommendation.views.status.Control.ping') def test_redis_status_pass(self, mock_ping): redis_status() self.assert_(True) - @patch('app.views.status.Control.ping') + @patch('recommendation.views.status.Control.ping') def test_redis_status_fail(self, mock_ping): mock_ping.side_effect = RedisError with self.assertRaises(ServiceDown): redis_status() - @patch('app.views.status.Control.ping') + @patch('recommendation.views.status.Control.ping') def test_celery_status_pass(self, mock_ping): mock_ping.return_value = MEMCACHED_PING_OK celery_status() self.assert_(True) - @patch('app.views.status.Control.ping') + @patch('recommendation.views.status.Control.ping') def test_celery_status_no_workers(self, mock_ping): mock_ping.return_value = MEMCACHED_PING_NO_WORKERS with self.assertRaises(ServiceDown): celery_status() - @patch('app.views.status.Control.ping') + @patch('recommendation.views.status.Control.ping') def test_celery_status_no_clusters(self, mock_ping): mock_ping.return_value = MEMCACHED_PING_NO_CLUSTERS with self.assertRaises(ServiceDown): celery_status() - @patch('app.views.status.Control.ping') + @patch('recommendation.views.status.Control.ping') def test_celery_status_workers_down(self, mock_ping): mock_ping.return_value = MEMCACHED_PING_BAD with self.assertRaises(ServiceDown): diff --git a/setup.cfg b/setup.cfg index 687fadc..9547bbb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,2 @@ [aliases] -test=nosetests --with-coverage --cover-package=app --cover-min-percentage=100 --exclude=app/src +test=nosetests --with-coverage --cover-package=recommendation --cover-min-percentage=100 --exclude=recommendation/src From 575a200bee356d68cde12480a16ab09d45ee66e5 Mon Sep 17 00:00:00 2001 From: Chuck Harmston Date: Thu, 25 Feb 2016 14:53:25 -0700 Subject: [PATCH 3/4] Fixes CI configuration. --- .coveragerc | 3 +++ .travis.yml | 4 ++-- circle.yml | 4 ++-- requirements.txt | 1 + 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.coveragerc b/.coveragerc index 5dbe770..8d53079 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,2 +1,5 @@ [run] source = app + +[report] +omit = recommendation/conf.py diff --git a/.travis.yml b/.travis.yml index 9f3b740..3c07af9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ language: python python: - "3.5" install: - - "pip install -r app/requirements.txt" + - "pip install -r requirements.txt" script: - - "nosetests --cover-min-percentage=100 --with-coverage --cover-package=app" + - "nosetests --with-coverage --cover-package=recommendation --exclude-dir=src" after_success: coveralls diff --git a/circle.yml b/circle.yml index bf67976..53618ee 100644 --- a/circle.yml +++ b/circle.yml @@ -18,7 +18,7 @@ dependencies: - printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s"}\n' "$CIRCLE_SHA1" "$CIRCLE_TAG" "$CIRCLE_PROJECT_USERNAME" "$CIRCLE_PROJECT_REPONAME" > version.json # build the actual deployment container - - docker build -t app:build app + - docker build -t app:build . # Clean up any old images; save the new one. - I="image-$(date +%j).tgz"; mkdir -p ~/docker; rm ~/docker/*; docker save app:build | gzip -c > ~/docker/$I; ls -l ~/docker @@ -27,7 +27,7 @@ dependencies: # Run flake8 and the Python test suite via nose. test: override: - - docker run app:build sh -c 'cd / && flake8 /app --exclude=/app/src && nosetests --exclude=/app/src' + - docker run -d app:build sh -c 'flake8 . --exclude=app/src && nosetests --exclude-dir=app/src' # Tag and push the container to Docker Hub. diff --git a/requirements.txt b/requirements.txt index 138e238..1cbe5e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,7 @@ Jinja2==2.8 MarkupSafe==0.23 mock==1.3.0 nose==1.3.7 +nose-exclude==0.4.1 oauth2==1.9.0.post1 redis==2.10.5 requests==2.9.1 From 132538f7342721f97d96547eac5c00924d5b4814 Mon Sep 17 00:00:00 2001 From: Chuck Harmston Date: Thu, 25 Feb 2016 17:44:32 -0700 Subject: [PATCH 4/4] Serves contribute.json with Flask. --- recommendation/factory.py | 2 ++ {conf => recommendation/static}/contribute.json | 0 recommendation/views/static.py | 11 +++++++++++ recommendation/views/tests/test_static.py | 15 +++++++++++++++ 4 files changed, 28 insertions(+) rename {conf => recommendation/static}/contribute.json (100%) create mode 100644 recommendation/views/static.py create mode 100644 recommendation/views/tests/test_static.py diff --git a/recommendation/factory.py b/recommendation/factory.py index a1f746d..6e39e73 100644 --- a/recommendation/factory.py +++ b/recommendation/factory.py @@ -4,6 +4,7 @@ from flask import Flask from recommendation import conf from recommendation.cors import cors_headers from recommendation.views.main import main +from recommendation.views.static import static from recommendation.views.status import status @@ -11,6 +12,7 @@ def create_app(): app = Flask(__name__) app.after_request(cors_headers) app.register_blueprint(main) + app.register_blueprint(static) app.register_blueprint(status) app.config.update( CELERY_BROKER_URL=conf.CELERY_BROKER_URL, diff --git a/conf/contribute.json b/recommendation/static/contribute.json similarity index 100% rename from conf/contribute.json rename to recommendation/static/contribute.json diff --git a/recommendation/views/static.py b/recommendation/views/static.py new file mode 100644 index 0000000..1adc8c9 --- /dev/null +++ b/recommendation/views/static.py @@ -0,0 +1,11 @@ +from os import path, pardir +from flask import Blueprint, send_file + + +static = Blueprint('static', __name__) +STATIC_DIR = path.join(path.dirname(path.abspath(__file__)), pardir, 'static') + + +@static.route('/contribute.json') +def lbheartbeat(): + return send_file(path.join(STATIC_DIR, 'contribute.json')) diff --git a/recommendation/views/tests/test_static.py b/recommendation/views/tests/test_static.py new file mode 100644 index 0000000..82cffef --- /dev/null +++ b/recommendation/views/tests/test_static.py @@ -0,0 +1,15 @@ +import json +from os import path + +from nose.tools import eq_ + +from recommendation.tests.util import AppTestCase +from recommendation.views.static import STATIC_DIR + + +class TestStaticViews(AppTestCase): + def test_contribute(self): + response = self.client.get('/contribute.json') + eq_(response.status_code, 200) + with open(path.join(STATIC_DIR, 'contribute.json')) as file_data: + eq_(json.load(file_data), response.json)