Merge pull request #52 from chuckharmston/51-single-container
Refactors into single-container application with supplementary services in compose (closes #51).
This commit is contained in:
Коммит
e1b6e759d8
|
@ -1,2 +1,5 @@
|
|||
[run]
|
||||
source = app
|
||||
|
||||
[report]
|
||||
omit = recommendation/conf.py
|
||||
|
|
|
@ -0,0 +1,9 @@
|
|||
__pycache__
|
||||
*.pyc
|
||||
src
|
||||
.DS_Store
|
||||
.eggs
|
||||
.git
|
||||
dist
|
||||
.coverage
|
||||
coverage.xml
|
|
@ -2,7 +2,7 @@ language: python
|
|||
python:
|
||||
- "3.5"
|
||||
install:
|
||||
- "pip install -r app/requirements.txt"
|
||||
- "pip install -r requirements.txt"
|
||||
script:
|
||||
- "nosetests --cover-min-percentage=100 --with-coverage --cover-package=app"
|
||||
- "nosetests --with-coverage --cover-package=recommendation --exclude-dir=src"
|
||||
after_success: coveralls
|
||||
|
|
|
@ -0,0 +1,19 @@
|
|||
# universal-search-recommendation
|
||||
#
|
||||
# VERSION 0.1
|
||||
|
||||
FROM python:3.5
|
||||
MAINTAINER https://mail.mozilla.org/listinfo/testpilot-dev
|
||||
|
||||
RUN groupadd --gid 10001 app && \
|
||||
useradd --uid 10001 --gid 10001 --shell /usr/sbin/nologin app
|
||||
|
||||
COPY ./requirements.txt /app/requirements.txt
|
||||
RUN pip install --upgrade --no-cache-dir -r /app/requirements.txt
|
||||
|
||||
COPY . /app
|
||||
ENV PYTHONPATH $PYTHONPATH:/app
|
||||
|
||||
USER app
|
||||
EXPOSE 8000
|
||||
ENTRYPOINT ["/app/conf/web.sh"]
|
|
@ -1,3 +0,0 @@
|
|||
__pycache__
|
||||
src
|
||||
.DS_Store
|
|
@ -1,19 +0,0 @@
|
|||
# universal-search-recommendation-app
|
||||
#
|
||||
# VERSION 0.1
|
||||
|
||||
FROM python:3.5
|
||||
MAINTAINER https://mail.mozilla.org/listinfo/testpilot-dev
|
||||
|
||||
RUN groupadd --gid 1001 app && \
|
||||
useradd --uid 1001 --gid 1001 --shell /usr/sbin/nologin app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY ./requirements.txt /app/requirements.txt
|
||||
RUN pip install --upgrade --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY . /app
|
||||
ENV PYTHONPATH $PYTHONPATH:/:/app
|
||||
|
||||
USER app
|
17
app/conf.py
17
app/conf.py
|
@ -1,17 +0,0 @@
|
|||
from os import environ as env
|
||||
|
||||
|
||||
CELERY_BROKER_URL = env.get('CELERY_BROKER_URL', 'redis://redis:6379/0')
|
||||
DEBUG = env.get('RECOMMENDATION_ENV', 'development') == 'development'
|
||||
KEY_PREFIX = env.get('RECOMMENDATION_KEY_PREFIX', 'query_')
|
||||
MEMCACHED_HOST = env.get('MEMCACHED_HOST', 'memcached:11211')
|
||||
|
||||
EMBEDLY_API_KEY = env.get('EMBEDLY_API_KEY', '')
|
||||
|
||||
# Yahoo BOSS API credentials.
|
||||
YAHOO_OAUTH_KEY = env.get('YAHOO_OAUTH_KEY', '')
|
||||
YAHOO_OAUTH_SECRET = env.get('YAHOO_OAUTH_SECRET', '')
|
||||
|
||||
# For non-Docker development server.
|
||||
HOST = env.get('RECOMMENDATION_HOST', '0.0.0.0')
|
||||
PORT = env.get('RECOMMENDATION_PORT', 5000)
|
|
@ -1,3 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
uwsgi --http :${PORT:-8000} --wsgi-file /app/main.py --master
|
|
@ -1,5 +0,0 @@
|
|||
from app import tasks # noqa
|
||||
from app.factory import create_app, create_queue
|
||||
|
||||
application = create_app()
|
||||
celery = create_queue()
|
|
@ -1,10 +0,0 @@
|
|||
from app.search.classification.domain import DomainClassifier
|
||||
from app.search.classification.embedly import EmbedlyClassifier
|
||||
from app.search.classification.wikipedia import WikipediaClassifier
|
||||
|
||||
|
||||
CLASSIFIERS = [
|
||||
DomainClassifier,
|
||||
EmbedlyClassifier,
|
||||
WikipediaClassifier,
|
||||
]
|
|
@ -1 +0,0 @@
|
|||
from app.tasks.task_recommend import recommend # noqa
|
|
@ -18,7 +18,7 @@ dependencies:
|
|||
- printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s"}\n' "$CIRCLE_SHA1" "$CIRCLE_TAG" "$CIRCLE_PROJECT_USERNAME" "$CIRCLE_PROJECT_REPONAME" > version.json
|
||||
|
||||
# build the actual deployment container
|
||||
- docker build -t app:build app
|
||||
- docker build -t app:build .
|
||||
|
||||
# Clean up any old images; save the new one.
|
||||
- I="image-$(date +%j).tgz"; mkdir -p ~/docker; rm ~/docker/*; docker save app:build | gzip -c > ~/docker/$I; ls -l ~/docker
|
||||
|
@ -27,7 +27,7 @@ dependencies:
|
|||
# Run flake8 and the Python test suite via nose.
|
||||
test:
|
||||
override:
|
||||
- docker run app:build sh -c 'cd / && flake8 /app --exclude=/app/src && nosetests --exclude=/app/src'
|
||||
- docker run -d app:build sh -c 'flake8 . --exclude=app/src && nosetests --exclude-dir=app/src'
|
||||
|
||||
|
||||
# Tag and push the container to Docker Hub.
|
||||
|
|
|
@ -1 +0,0 @@
|
|||
.DS_Store
|
|
@ -1,12 +0,0 @@
|
|||
# universal-search-recommendation-proxy
|
||||
#
|
||||
# VERSION 0.1
|
||||
|
||||
FROM nginx:1.9.9
|
||||
MAINTAINER https://mail.mozilla.org/listinfo/testpilot-dev
|
||||
|
||||
RUN rm /etc/nginx/conf.d/default.conf
|
||||
COPY recommendation.conf /etc/nginx/conf.d/
|
||||
|
||||
RUN mkdir -p /var/www/
|
||||
COPY contribute.json /var/www/
|
|
@ -1,16 +0,0 @@
|
|||
server {
|
||||
listen 80;
|
||||
server_name "";
|
||||
charset utf-8;
|
||||
|
||||
location /contribute.json {
|
||||
alias /var/www/contribute.json;
|
||||
}
|
||||
|
||||
location / {
|
||||
proxy_pass http://app:8000;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,3 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
uwsgi --http :${PORT:-8000} --wsgi-file /app/recommendation/wsgi.py --master
|
|
@ -0,0 +1,3 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
celery worker --app=recommendation
|
|
@ -1,37 +1,26 @@
|
|||
app:
|
||||
build: ./app
|
||||
command: uwsgi --http :8000 --wsgi-file /app/main.py --master
|
||||
env_file: .env
|
||||
expose:
|
||||
- "8000"
|
||||
links:
|
||||
- memcached
|
||||
- redis
|
||||
restart: always
|
||||
|
||||
nginx:
|
||||
build: ./conf/nginx
|
||||
links:
|
||||
- app
|
||||
ports:
|
||||
- "80:80"
|
||||
restart: always
|
||||
|
||||
memcached:
|
||||
image: memcached
|
||||
restart: always
|
||||
expose:
|
||||
- "11211"
|
||||
image: memcached
|
||||
restart: always
|
||||
ports:
|
||||
- "11211"
|
||||
- "11211:11211"
|
||||
|
||||
redis:
|
||||
expose:
|
||||
- "6379"
|
||||
image: redis
|
||||
ports:
|
||||
- "6379"
|
||||
- "6379:6379"
|
||||
|
||||
celery:
|
||||
build: ./app
|
||||
command: celery worker --app=app.main
|
||||
worker:
|
||||
build: ./
|
||||
entrypoint: /app/conf/worker.sh
|
||||
env_file: .env
|
||||
environment:
|
||||
- CELERY_BROKER_URL=redis://redis:6379/0
|
||||
- RECOMMENDATION_ENV="worker"
|
||||
links:
|
||||
- memcached
|
||||
- redis
|
||||
|
|
|
@ -1,40 +1,86 @@
|
|||
# Local Development
|
||||
## Local Development
|
||||
|
||||
A multiple-container Docker configuration is maintained to support local development.
|
||||
|
||||
|
||||
## First-time setup
|
||||
### Configuration
|
||||
|
||||
A few steps are required the first time
|
||||
Some environment variables are required. Each of those are contained within `.env.dist`, which should be copied to `.env` and populated as appropriate.
|
||||
|
||||
|
||||
### Services
|
||||
|
||||
`universal-search-recommendation` requires several additional services: a [Memcached](http://memcached.org/) cache backend, a [Celery](http://www.celeryproject.org/) task queue, and a [Redis](http://redis.io/) Celery backend. A [Compose](https://docs.docker.com/compose/) configuration is included to streamline setup and management of those services.
|
||||
|
||||
|
||||
#### Set up Docker host.
|
||||
|
||||
The first time you set up the services, you must create the Docker host.
|
||||
|
||||
- [Install Docker Toolbox](https://www.docker.com/products/docker-toolbox).
|
||||
- Some environment variables are required. Each of those are contained within `.env.dist`, which should be copied to `.env` and populated as appropriate.
|
||||
- Create a Docker host:
|
||||
|
||||
```bash
|
||||
docker-machine create --driver virtualbox universal-search
|
||||
docker-machine create --driver virtualbox universal-search-dev
|
||||
```
|
||||
|
||||
- Add an entry to your hosts file:
|
||||
- Add an entry to your `hosts` file:
|
||||
|
||||
```bash
|
||||
sudo sh -c "echo $(docker-machine ip universal-search 2>/dev/null) universal-search.dev >> /etc/hosts"
|
||||
sudo sh -c "echo $(docker-machine ip universal-search-dev 2>/dev/null) universal-search.dev >> /etc/hosts"
|
||||
```
|
||||
|
||||
|
||||
## Start machine and containers
|
||||
#### Start services.
|
||||
|
||||
The machine and containers can then be started at any time:
|
||||
After the host is set up, you can run each service by:
|
||||
|
||||
```
|
||||
docker-machine start universal-search-dev
|
||||
eval $(docker-machine env universal-search-dev)
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
To verify that each service is up, run:
|
||||
|
||||
```bash
|
||||
docker-machine start universal-search
|
||||
eval $(docker-machine env universal-search)
|
||||
docker-compose up
|
||||
docker-compose ps
|
||||
```
|
||||
|
||||
While `docker-compose up` is running, the recommendation server may be accessed at [http://universal-search.dev](http://universal-search.dev).
|
||||
In the output you should see containers named `recommendation_memcached_1`, `recommendation_redis_1`, and `recommendation_worker_1`. The state for each of these should be `Up`.
|
||||
|
||||
|
||||
## Architecture
|
||||
### Application
|
||||
|
||||
An nginx server is the entry point, acting as a reverse proxy to forward requests to a Python container running uWSGI, which manages the application server. Memcached, Celery, and Redis containers run parallel; used for caching, as a task queue, and as a backend for the task queue, respectively.
|
||||
`universal-search-recommendation` is managed as a Docker container that is separately built from the services, but run on the same host as the services.
|
||||
|
||||
To build it:
|
||||
|
||||
```bash
|
||||
docker build -t universal-search-recommendation .
|
||||
```
|
||||
|
||||
To run it:
|
||||
|
||||
```bash
|
||||
docker run -d -e "RECOMMENDATION_SERVICES=`docker-machine ip universal-search-dev`" -p 80:8000/tcp universal-search-recommendation
|
||||
```
|
||||
|
||||
To verify that the application is running:
|
||||
|
||||
```bash
|
||||
docker ps --filter ancestor="universal-search-recommendation"
|
||||
```
|
||||
|
||||
You should see one container running, and its status should begin with `Up`.
|
||||
|
||||
|
||||
### Usage
|
||||
|
||||
If the application and services are all running, you should be able to access to access the recommendation server at [http://universal-search.dev](http://universal-search.dev). After making changes to the application, you will need to stop it:
|
||||
|
||||
```bash
|
||||
docker stop $(docker ps --filter ancestor="universal-search-recommendation" --format="{{.ID}}")
|
||||
```
|
||||
|
||||
Then rebuild and start it back up, per above.
|
||||
|
|
|
@ -0,0 +1,4 @@
|
|||
from recommendation import tasks # noqa
|
||||
from recommendation.factory import create_queue
|
||||
|
||||
celery = create_queue()
|
|
@ -0,0 +1,17 @@
|
|||
from os import environ as env
|
||||
|
||||
|
||||
DEBUG = env.get('RECOMMENDATION_ENV', 'development') == 'development'
|
||||
KEY_PREFIX = env.get('RECOMMENDATION_KEY_PREFIX', 'query_')
|
||||
|
||||
EMBEDLY_API_KEY = env.get('EMBEDLY_API_KEY', '')
|
||||
YAHOO_OAUTH_KEY = env.get('YAHOO_OAUTH_KEY', '')
|
||||
YAHOO_OAUTH_SECRET = env.get('YAHOO_OAUTH_SECRET', '')
|
||||
|
||||
RECOMMENDATION_SERVICES = env.get('RECOMMENDATION_SERVICES')
|
||||
if DEBUG and RECOMMENDATION_SERVICES:
|
||||
CELERY_BROKER_URL = 'redis://%s:6379/0' % RECOMMENDATION_SERVICES
|
||||
MEMCACHED_HOST = '%s:11211' % RECOMMENDATION_SERVICES
|
||||
else:
|
||||
CELERY_BROKER_URL = env.get('CELERY_BROKER_URL', 'redis://redis:6379/0')
|
||||
MEMCACHED_HOST = env.get('MEMCACHED_HOST', 'memcached:11211')
|
|
@ -1,16 +1,18 @@
|
|||
from celery import Celery
|
||||
from flask import Flask
|
||||
|
||||
from app import conf
|
||||
from app.cors import cors_headers
|
||||
from app.views.main import main
|
||||
from app.views.status import status
|
||||
from recommendation import conf
|
||||
from recommendation.cors import cors_headers
|
||||
from recommendation.views.main import main
|
||||
from recommendation.views.static import static
|
||||
from recommendation.views.status import status
|
||||
|
||||
|
||||
def create_app():
|
||||
app = Flask(__name__)
|
||||
app.after_request(cors_headers)
|
||||
app.register_blueprint(main)
|
||||
app.register_blueprint(static)
|
||||
app.register_blueprint(status)
|
||||
app.config.update(
|
||||
CELERY_BROKER_URL=conf.CELERY_BROKER_URL,
|
|
@ -0,0 +1,5 @@
|
|||
from recommendation import tasks # noqa
|
||||
from recommendation.factory import create_app, create_queue
|
||||
|
||||
application = create_app()
|
||||
celery = create_queue()
|
|
@ -1,6 +1,6 @@
|
|||
from memcache import Client
|
||||
|
||||
from app import conf
|
||||
from recommendation import conf
|
||||
|
||||
|
||||
memcached = Client([conf.MEMCACHED_HOST])
|
|
@ -5,7 +5,7 @@ from inspect import getargspec
|
|||
|
||||
from wrapt import ObjectProxy
|
||||
|
||||
from app.memcached import memcached
|
||||
from recommendation.memcached import memcached
|
||||
|
||||
|
||||
class MemorizedObject(ObjectProxy):
|
|
@ -0,0 +1,10 @@
|
|||
from recommendation.search.classification.domain import DomainClassifier
|
||||
from recommendation.search.classification.embedly import EmbedlyClassifier
|
||||
from recommendation.search.classification.wikipedia import WikipediaClassifier
|
||||
|
||||
|
||||
CLASSIFIERS = [
|
||||
DomainClassifier,
|
||||
EmbedlyClassifier,
|
||||
WikipediaClassifier,
|
||||
]
|
|
@ -1,4 +1,4 @@
|
|||
from app.search.classification.base import BaseClassifier
|
||||
from recommendation.search.classification.base import BaseClassifier
|
||||
|
||||
|
||||
class DomainClassifier(BaseClassifier):
|
|
@ -2,11 +2,10 @@ from urllib.parse import urlencode
|
|||
|
||||
import requests
|
||||
|
||||
import app.conf
|
||||
from app.memorize import memorize
|
||||
|
||||
from app.search.classification.base import BaseClassifier
|
||||
from app.search.classification.wikipedia import WikipediaClassifier
|
||||
from recommendation import conf
|
||||
from recommendation.memorize import memorize
|
||||
from recommendation.search.classification.base import BaseClassifier
|
||||
from recommendation.search.classification.wikipedia import WikipediaClassifier
|
||||
|
||||
|
||||
class EmbedlyClassifier(BaseClassifier):
|
||||
|
@ -51,7 +50,7 @@ class EmbedlyClassifier(BaseClassifier):
|
|||
|
||||
def _api_url(self, url):
|
||||
return '%s?%s' % (self.api_url, urlencode({
|
||||
'key': app.conf.EMBEDLY_API_KEY,
|
||||
'key': conf.EMBEDLY_API_KEY,
|
||||
'words': 20,
|
||||
'secure': 'true',
|
||||
'url': url
|
|
@ -1,6 +1,6 @@
|
|||
from unittest import TestCase
|
||||
|
||||
from app.search.classification.base import BaseClassifier
|
||||
from recommendation.search.classification.base import BaseClassifier
|
||||
|
||||
|
||||
class TestBaseClassifier(TestCase):
|
|
@ -3,7 +3,7 @@ from urllib.parse import ParseResult
|
|||
|
||||
from nose.tools import eq_, ok_
|
||||
|
||||
from app.search.classification.domain import DomainClassifier
|
||||
from recommendation.search.classification.domain import DomainClassifier
|
||||
|
||||
|
||||
DOMAIN = 'www.mozilla.com'
|
|
@ -5,8 +5,8 @@ from urllib.parse import parse_qs, urlparse
|
|||
import responses
|
||||
from nose.tools import eq_, ok_
|
||||
|
||||
from app.search.classification.embedly import EmbedlyClassifier
|
||||
from app.tests.memcached import mock_memcached
|
||||
from recommendation.search.classification.embedly import EmbedlyClassifier
|
||||
from recommendation.tests.memcached import mock_memcached
|
||||
|
||||
|
||||
MOCK_API_KEY = '0123456789abcdef'
|
||||
|
@ -92,7 +92,7 @@ class TestEmbedlyClassifier(TestCase):
|
|||
eq_(self._matches('https://en.wikipedia.org/wiki/Mozilla'), True)
|
||||
eq_(self._matches('https://en.wikipedia.org/wiki/Mozilla/'), True)
|
||||
|
||||
@patch('app.conf.EMBEDLY_API_KEY', MOCK_API_KEY)
|
||||
@patch('recommendation.conf.EMBEDLY_API_KEY', MOCK_API_KEY)
|
||||
def test_api_url(self):
|
||||
api_url = self._api_url(MOCK_RESULT_URL)
|
||||
api_qs = parse_qs(urlparse(api_url).query)
|
||||
|
@ -101,8 +101,9 @@ class TestEmbedlyClassifier(TestCase):
|
|||
eq_(api_qs['url'][0], MOCK_RESULT_URL)
|
||||
eq_(api_qs['secure'][0], 'true')
|
||||
|
||||
@patch('app.memorize.memcached', mock_memcached)
|
||||
@patch('app.search.classification.embedly.EmbedlyClassifier._api_url')
|
||||
@patch('recommendation.memorize.memcached', mock_memcached)
|
||||
@patch('recommendation.search.classification.embedly.EmbedlyClassifier'
|
||||
'._api_url')
|
||||
@responses.activate
|
||||
def test_api_response(self, mock_api_url):
|
||||
mock_api_url.return_value = MOCK_API_URL
|
||||
|
@ -116,8 +117,9 @@ class TestEmbedlyClassifier(TestCase):
|
|||
eq_(response_cold.cache_key, response_warm.cache_key)
|
||||
eq_(response_cold, response_warm, MOCK_RESPONSE)
|
||||
|
||||
@patch('app.memorize.memcached', mock_memcached)
|
||||
@patch('app.search.classification.embedly.EmbedlyClassifier._api_url')
|
||||
@patch('recommendation.memorize.memcached', mock_memcached)
|
||||
@patch('recommendation.search.classification.embedly.EmbedlyClassifier'
|
||||
'._api_url')
|
||||
@responses.activate
|
||||
def test_enhance(self, mock_api_url):
|
||||
mock_api_url.return_value = MOCK_API_URL
|
||||
|
@ -130,8 +132,10 @@ class TestEmbedlyClassifier(TestCase):
|
|||
eq_(enhanced['favicon']['colors'], MOCK_RESPONSE['favicon_colors'])
|
||||
eq_(enhanced['favicon']['url'], MOCK_RESPONSE['favicon_url'])
|
||||
|
||||
@patch('app.search.classification.embedly.EmbedlyClassifier._api_response')
|
||||
@patch('app.search.classification.embedly.EmbedlyClassifier._get_image')
|
||||
@patch('recommendation.search.classification.embedly.EmbedlyClassifier'
|
||||
'._api_response')
|
||||
@patch('recommendation.search.classification.embedly.EmbedlyClassifier'
|
||||
'._get_image')
|
||||
def test_enhance_no_images(self, mock_get_image, mock_api_response):
|
||||
mock_api_response.return_value = MOCK_RESPONSE
|
||||
mock_get_image.side_effect = KeyError
|
|
@ -5,8 +5,8 @@ from urllib.parse import parse_qs, urlparse
|
|||
import responses
|
||||
from nose.tools import eq_, ok_
|
||||
|
||||
from app.search.classification.wikipedia import WikipediaClassifier
|
||||
from app.tests.memcached import mock_memcached
|
||||
from recommendation.search.classification.wikipedia import WikipediaClassifier
|
||||
from recommendation.tests.memcached import mock_memcached
|
||||
|
||||
|
||||
SLUG = 'Mozilla'
|
||||
|
@ -70,8 +70,9 @@ class TestWikipediaClassifier(TestCase):
|
|||
actual[4] = parse_qs(actual[4])
|
||||
eq_(expected, actual)
|
||||
|
||||
@patch('app.memorize.memcached', mock_memcached)
|
||||
@patch('app.search.classification.wikipedia.WikipediaClassifier._api_url')
|
||||
@patch('recommendation.memorize.memcached', mock_memcached)
|
||||
@patch('recommendation.search.classification.wikipedia.WikipediaClassifier'
|
||||
'._api_url')
|
||||
@responses.activate
|
||||
def test_api_response(self, mock_api_url):
|
||||
"""
|
||||
|
@ -94,8 +95,9 @@ class TestWikipediaClassifier(TestCase):
|
|||
eq_(response_cold.cache_key, response_warm.cache_key)
|
||||
eq_(response_cold, response_warm, EXPECTED_SANITIZED_RESPONSE)
|
||||
|
||||
@patch('app.memorize.memcached', mock_memcached)
|
||||
@patch('app.search.classification.wikipedia.WikipediaClassifier._api_url')
|
||||
@patch('recommendation.memorize.memcached', mock_memcached)
|
||||
@patch('recommendation.search.classification.wikipedia.WikipediaClassifier'
|
||||
'._api_url')
|
||||
@responses.activate
|
||||
def test_enhance(self, mock_api_url):
|
||||
"""
|
|
@ -2,8 +2,8 @@ from urllib.parse import urlencode, urlunparse
|
|||
|
||||
import requests
|
||||
|
||||
from app.memorize import memorize
|
||||
from app.search.classification.base import BaseClassifier
|
||||
from recommendation.memorize import memorize
|
||||
from recommendation.search.classification.base import BaseClassifier
|
||||
|
||||
|
||||
class WikipediaClassifier(BaseClassifier):
|
|
@ -1,6 +1,6 @@
|
|||
import bleach
|
||||
|
||||
from app.memorize import memorize
|
||||
from recommendation.memorize import memorize
|
||||
|
||||
|
||||
class BaseQueryEngine(object):
|
|
@ -3,8 +3,8 @@ from unittest.mock import patch
|
|||
|
||||
from nose.tools import eq_, ok_
|
||||
|
||||
from app.search.query.base import BaseQueryEngine
|
||||
from app.tests.memcached import mock_memcached
|
||||
from recommendation.search.query.base import BaseQueryEngine
|
||||
from recommendation.tests.memcached import mock_memcached
|
||||
|
||||
|
||||
QUERY = 'loveland'
|
||||
|
@ -41,9 +41,10 @@ class TestBaseQueryEngine(TestCase):
|
|||
def test_get_best_result(self):
|
||||
eq_(self.instance.get_best_result(MOCK_RESULTS), MOCK_RESULTS[0])
|
||||
|
||||
@patch('app.search.query.base.BaseQueryEngine.get_result_abstract')
|
||||
@patch('app.search.query.base.BaseQueryEngine.get_result_title')
|
||||
@patch('app.search.query.base.BaseQueryEngine.get_result_url')
|
||||
@patch('recommendation.search.query.base.BaseQueryEngine'
|
||||
'.get_result_abstract')
|
||||
@patch('recommendation.search.query.base.BaseQueryEngine.get_result_title')
|
||||
@patch('recommendation.search.query.base.BaseQueryEngine.get_result_url')
|
||||
def test_sanitize_result(self, *args):
|
||||
result = self.instance.sanitize_result(MOCK_RESULTS[0])
|
||||
ok_(all(k in result.keys() for k in ['abstract', 'title', 'url']))
|
||||
|
@ -69,8 +70,8 @@ class TestBaseQueryEngine(TestCase):
|
|||
eq_(self.instance.get_result_abstract(MOCK_RESULTS[0]),
|
||||
MOCK_RESULT_SANITIZED['abstract'])
|
||||
|
||||
@patch('app.search.query.base.BaseQueryEngine.fetch')
|
||||
@patch('app.memorize.memcached', mock_memcached)
|
||||
@patch('recommendation.search.query.base.BaseQueryEngine.fetch')
|
||||
@patch('recommendation.memorize.memcached', mock_memcached)
|
||||
def test_search(self, mock_fetch):
|
||||
mock_fetch.return_value = MOCK_RESULTS
|
||||
results_cold = self.instance.search(QUERY)
|
|
@ -5,8 +5,8 @@ from urllib.parse import parse_qs, urlparse
|
|||
import responses
|
||||
from nose.tools import eq_, ok_
|
||||
|
||||
from app.search.query.yahoo import YahooQueryEngine
|
||||
from app.tests.memcached import mock_memcached
|
||||
from recommendation.search.query.yahoo import YahooQueryEngine
|
||||
from recommendation.tests.memcached import mock_memcached
|
||||
|
||||
|
||||
QUERY = 'loveland'
|
||||
|
@ -56,7 +56,7 @@ class TestYahooClassifier(TestCase):
|
|||
ok_(len(query_string['oauth_version']))
|
||||
eq_(query_string['q'][0], QUERY)
|
||||
|
||||
@patch('app.memorize.memcached', mock_memcached)
|
||||
@patch('recommendation.memorize.memcached', mock_memcached)
|
||||
@responses.activate
|
||||
def test_fetch(self):
|
||||
responses.add(responses.GET, YahooQueryEngine.url,
|
||||
|
@ -68,7 +68,7 @@ class TestYahooClassifier(TestCase):
|
|||
eq_(response_cold.cache_key, response_warm.cache_key)
|
||||
eq_(response_cold, response_warm, MOCK_RESPONSE)
|
||||
|
||||
@patch('app.memorize.memcached', mock_memcached)
|
||||
@patch('recommendation.memorize.memcached', mock_memcached)
|
||||
@responses.activate
|
||||
def test_sanitize_response(self):
|
||||
responses.add(responses.GET, YahooQueryEngine.url,
|
|
@ -4,9 +4,9 @@ from urllib.parse import quote_plus
|
|||
import oauth2
|
||||
import requests
|
||||
|
||||
from app import conf
|
||||
from app.memorize import memorize
|
||||
from app.search.query.base import BaseQueryEngine
|
||||
from recommendation import conf
|
||||
from recommendation.memorize import memorize
|
||||
from recommendation.search.query.base import BaseQueryEngine
|
||||
|
||||
|
||||
class YahooQueryEngine(BaseQueryEngine):
|
|
@ -1,6 +1,6 @@
|
|||
from app.search.classification import CLASSIFIERS
|
||||
from app.search.query.yahoo import YahooQueryEngine
|
||||
from app.search.suggest.bing import BingSuggestionEngine
|
||||
from recommendation.search.classification import CLASSIFIERS
|
||||
from recommendation.search.query.yahoo import YahooQueryEngine
|
||||
from recommendation.search.suggest.bing import BingSuggestionEngine
|
||||
|
||||
|
||||
class SearchRecommendation(object):
|
|
@ -1,4 +1,4 @@
|
|||
from app.memorize import memorize
|
||||
from recommendation.memorize import memorize
|
||||
|
||||
|
||||
class BaseSuggestionEngine(object):
|
|
@ -1,7 +1,7 @@
|
|||
import requests
|
||||
|
||||
from app.memorize import memorize
|
||||
from app.search.suggest.base import BaseSuggestionEngine
|
||||
from recommendation.memorize import memorize
|
||||
from recommendation.search.suggest.base import BaseSuggestionEngine
|
||||
|
||||
|
||||
class BingSuggestionEngine(BaseSuggestionEngine):
|
|
@ -3,8 +3,8 @@ from unittest import TestCase
|
|||
from mock import patch
|
||||
from nose.tools import eq_, ok_
|
||||
|
||||
from app.search.suggest.base import BaseSuggestionEngine
|
||||
from app.tests.memcached import mock_memcached
|
||||
from recommendation.search.suggest.base import BaseSuggestionEngine
|
||||
from recommendation.tests.memcached import mock_memcached
|
||||
|
||||
|
||||
QUERY = ''
|
||||
|
@ -29,8 +29,8 @@ class TestBaseSuggestionEngine(TestCase):
|
|||
def test_sanitize(self):
|
||||
eq_(self.instance.sanitize(RESULTS), RESULTS)
|
||||
|
||||
@patch('app.search.suggest.base.BaseSuggestionEngine.fetch')
|
||||
@patch('app.memorize.memcached', mock_memcached)
|
||||
@patch('recommendation.search.suggest.base.BaseSuggestionEngine.fetch')
|
||||
@patch('recommendation.memorize.memcached', mock_memcached)
|
||||
def test_search(self, mock_fetch):
|
||||
mock_fetch.return_value = RESULTS
|
||||
results_cold = self.instance.search(QUERY)
|
|
@ -4,8 +4,8 @@ import responses
|
|||
from mock import patch
|
||||
from nose.tools import eq_, ok_
|
||||
|
||||
from app.search.suggest.bing import BingSuggestionEngine
|
||||
from app.tests.memcached import mock_memcached
|
||||
from recommendation.search.suggest.bing import BingSuggestionEngine
|
||||
from recommendation.tests.memcached import mock_memcached
|
||||
|
||||
|
||||
QUERY = 'original query'
|
||||
|
@ -23,7 +23,7 @@ class TestBingSuggestionEngine(TestCase):
|
|||
def setUp(self):
|
||||
self.instance = BingSuggestionEngine(QUERY)
|
||||
|
||||
@patch('app.memorize.memcached', mock_memcached)
|
||||
@patch('recommendation.memorize.memcached', mock_memcached)
|
||||
@responses.activate
|
||||
def test_fetch(self):
|
||||
responses.add(responses.GET, BingSuggestionEngine.url, json=RESPONSE,
|
|
@ -3,18 +3,18 @@ from unittest import TestCase
|
|||
from mock import patch
|
||||
from nose.tools import eq_, ok_
|
||||
|
||||
from app.search.classification.domain import DomainClassifier
|
||||
from app.search.classification.tests.test_domain import DOMAIN
|
||||
from app.search.recommendation import SearchRecommendation
|
||||
from app.search.suggest.base import BaseSuggestionEngine
|
||||
from app.search.suggest.bing import BingSuggestionEngine
|
||||
from app.search.suggest.tests.test_bing import (QUERY as BING_QUERY,
|
||||
RESULTS as BING_RESULTS)
|
||||
from app.search.query.base import BaseQueryEngine
|
||||
from app.search.query.yahoo import YahooQueryEngine
|
||||
from app.search.query.tests.test_yahoo import (QUERY as YAHOO_QUERY,
|
||||
MOCK_RESPONSE as YAHOO_RESPONSE)
|
||||
from app.tests.memcached import mock_memcached
|
||||
from recommendation.search.classification.domain import DomainClassifier
|
||||
from recommendation.search.classification.tests.test_domain import DOMAIN
|
||||
from recommendation.search.recommendation import SearchRecommendation
|
||||
from recommendation.search.suggest.base import BaseSuggestionEngine
|
||||
from recommendation.search.suggest.bing import BingSuggestionEngine
|
||||
from recommendation.search.suggest.tests.test_bing import (
|
||||
QUERY as BING_QUERY, RESULTS as BING_RESULTS)
|
||||
from recommendation.search.query.base import BaseQueryEngine
|
||||
from recommendation.search.query.yahoo import YahooQueryEngine
|
||||
from recommendation.search.query.tests.test_yahoo import (
|
||||
QUERY as YAHOO_QUERY, MOCK_RESPONSE as YAHOO_RESPONSE)
|
||||
from recommendation.tests.memcached import mock_memcached
|
||||
|
||||
|
||||
QUERY = 'Cubs'
|
||||
|
@ -35,7 +35,8 @@ class TestSearchRecommendation(TestCase):
|
|||
engine = self.instance.get_query_engine()
|
||||
ok_(issubclass(engine, BaseQueryEngine))
|
||||
|
||||
@patch('app.search.classification.domain.DomainClassifier.is_match')
|
||||
@patch('recommendation.search.classification.domain.DomainClassifier'
|
||||
'.is_match')
|
||||
def test_get_classifiers(self, mock_match):
|
||||
mock_match.return_value = True
|
||||
classifiers = self.instance.get_classifiers({
|
||||
|
@ -45,9 +46,9 @@ class TestSearchRecommendation(TestCase):
|
|||
ok_(isinstance(classifiers[0], DomainClassifier))
|
||||
return classifiers
|
||||
|
||||
@patch(('app.search.recommendation.SearchRecommendation.get_suggestion_eng'
|
||||
'ine'))
|
||||
@patch('app.search.suggest.bing.BingSuggestionEngine.search')
|
||||
@patch(('recommendation.search.recommendation.SearchRecommendation'
|
||||
'.get_suggestion_engine'))
|
||||
@patch('recommendation.search.suggest.bing.BingSuggestionEngine.search')
|
||||
def test_get_suggestions(self, mock_bing, mock_suggestion_engine):
|
||||
mock_bing.return_value = BING_RESULTS
|
||||
mock_suggestion_engine.return_value = BingSuggestionEngine
|
||||
|
@ -57,8 +58,9 @@ class TestSearchRecommendation(TestCase):
|
|||
def test_get_top_suggestion(self):
|
||||
eq_(self.instance.get_top_suggestion(BING_RESULTS), BING_RESULTS[0])
|
||||
|
||||
@patch('app.search.recommendation.SearchRecommendation.get_query_engine')
|
||||
@patch('app.search.query.yahoo.YahooQueryEngine.search')
|
||||
@patch('recommendation.search.recommendation.SearchRecommendation'
|
||||
'.get_query_engine')
|
||||
@patch('recommendation.search.query.yahoo.YahooQueryEngine.search')
|
||||
def test_do_query(self, mock_yahoo, mock_query_engine):
|
||||
response = YAHOO_RESPONSE['bossresponse']['web']['results'][0]
|
||||
mock_yahoo.return_value = response
|
||||
|
@ -66,11 +68,15 @@ class TestSearchRecommendation(TestCase):
|
|||
eq_(self.instance.do_query(YAHOO_QUERY), response)
|
||||
eq_(mock_yahoo.call_count, 1)
|
||||
|
||||
@patch('app.search.recommendation.SearchRecommendation.get_classifiers')
|
||||
@patch('app.search.recommendation.SearchRecommendation.do_query')
|
||||
@patch('app.search.recommendation.SearchRecommendation.get_top_suggestion')
|
||||
@patch('app.search.recommendation.SearchRecommendation.get_suggestions')
|
||||
@patch('app.memorize.memcached', mock_memcached)
|
||||
@patch('recommendation.search.recommendation.SearchRecommendation'
|
||||
'.get_classifiers')
|
||||
@patch('recommendation.search.recommendation.SearchRecommendation'
|
||||
'.do_query')
|
||||
@patch('recommendation.search.recommendation.SearchRecommendation'
|
||||
'.get_top_suggestion')
|
||||
@patch('recommendation.search.recommendation.SearchRecommendation'
|
||||
'.get_suggestions')
|
||||
@patch('recommendation.memorize.memcached', mock_memcached)
|
||||
def test_do_search_get_recommendation(self, mock_suggestions,
|
||||
mock_top_suggestion, mock_result,
|
||||
mock_classifiers):
|
|
@ -0,0 +1 @@
|
|||
from recommendation.tasks.task_recommend import recommend # noqa
|
|
@ -1,6 +1,6 @@
|
|||
from app.factory import create_queue
|
||||
from app.memcached import memcached
|
||||
from app.search.recommendation import SearchRecommendation
|
||||
from recommendation.factory import create_queue
|
||||
from recommendation.memcached import memcached
|
||||
from recommendation.search.recommendation import SearchRecommendation
|
||||
|
||||
|
||||
queue = create_queue()
|
|
@ -3,7 +3,7 @@ from unittest import TestCase
|
|||
from mock import patch
|
||||
from nose.tools import eq_
|
||||
|
||||
from app.tests.memcached import mock_memcached
|
||||
from recommendation.tests.memcached import mock_memcached
|
||||
|
||||
|
||||
KEY = 'query_key'
|
||||
|
@ -13,12 +13,13 @@ RESULTS = {
|
|||
}
|
||||
|
||||
|
||||
@patch('app.tasks.task_recommend.memcached', mock_memcached)
|
||||
@patch('recommendation.tasks.task_recommend.memcached', mock_memcached)
|
||||
class TestRecommendTask(TestCase):
|
||||
|
||||
@patch('app.search.recommendation.SearchRecommendation.do_search')
|
||||
@patch('recommendation.search.recommendation.SearchRecommendation'
|
||||
'.do_search')
|
||||
def test_recommend(self, mock_do_search):
|
||||
from app.tasks.task_recommend import recommend
|
||||
from recommendation.tasks.task_recommend import recommend
|
||||
mock_do_search.return_value = RESULTS
|
||||
results = recommend.apply(args=[QUERY, KEY]).get()
|
||||
eq_(results, RESULTS)
|
|
@ -5,12 +5,12 @@ class FakeMemcached(object):
|
|||
`memorize` decorator like so:
|
||||
|
||||
from unittest.mock import patch
|
||||
from app.tests.memcached import mock_memcached
|
||||
from recommendation.tests.memcached import mock_memcached
|
||||
|
||||
def tearDown(self):
|
||||
mock_memcached.flush_all()
|
||||
|
||||
@patch('app.memorize.memcached', mock_memcached)
|
||||
@patch('recommendation.memorize.memcached', mock_memcached)
|
||||
def test_foo(self):
|
||||
assert(True)
|
||||
"""
|
|
@ -0,0 +1,19 @@
|
|||
from unittest import TestCase
|
||||
|
||||
from celery.app.base import Celery
|
||||
from nose.tools import eq_, ok_
|
||||
|
||||
from recommendation.celery import celery
|
||||
|
||||
|
||||
class TestCeleryApp(TestCase):
|
||||
def test_celery_app(self):
|
||||
from recommendation import celery as celery_module
|
||||
ok_(hasattr(celery_module, 'celery'))
|
||||
|
||||
def test_celery_app_type(self):
|
||||
eq_(type(celery), Celery)
|
||||
|
||||
def test_celery_app_singleton(self):
|
||||
from recommendation.celery import celery as celery_2
|
||||
eq_(id(celery), id(celery_2))
|
|
@ -1,7 +1,7 @@
|
|||
from nose.tools import ok_
|
||||
|
||||
from app.cors import cors_headers
|
||||
from app.tests.util import AppTestCase
|
||||
from recommendation.cors import cors_headers
|
||||
from recommendation.tests.util import AppTestCase
|
||||
|
||||
|
||||
class TestCORS(AppTestCase):
|
|
@ -4,16 +4,16 @@ from celery.app.base import Celery
|
|||
from mock import patch
|
||||
from nose.tools import eq_, ok_
|
||||
|
||||
from app.cors import cors_headers
|
||||
from app.factory import create_app, create_queue
|
||||
from app.tests.util import AppTestCase
|
||||
from recommendation.cors import cors_headers
|
||||
from recommendation.factory import create_app, create_queue
|
||||
from recommendation.tests.util import AppTestCase
|
||||
|
||||
|
||||
BROKER_URL = 'http://celery.carrots/broccoli'
|
||||
|
||||
|
||||
class TestCreateApp(TestCase):
|
||||
@patch('app.factory.conf.CELERY_BROKER_URL', BROKER_URL)
|
||||
@patch('recommendation.factory.conf.CELERY_BROKER_URL', BROKER_URL)
|
||||
def test_create_app(self):
|
||||
app = create_app()
|
||||
ok_(cors_headers in app.after_request_funcs[None])
|
|
@ -4,8 +4,8 @@ from flask.ext.testing import TestCase as FlaskTestCase
|
|||
from mock import patch
|
||||
from nose.tools import eq_, ok_
|
||||
|
||||
from app.cors import cors_headers
|
||||
from app.tests.memcached import mock_memcached
|
||||
from recommendation.cors import cors_headers
|
||||
from recommendation.tests.memcached import mock_memcached
|
||||
|
||||
|
||||
KEY = 'query_key'
|
||||
|
@ -17,7 +17,7 @@ EXCEPTION_ARGS = ['args']
|
|||
EXCEPTION = RuntimeError(*EXCEPTION_ARGS)
|
||||
|
||||
|
||||
@patch('app.tasks.task_recommend.memcached', mock_memcached)
|
||||
@patch('recommendation.tasks.task_recommend.memcached', mock_memcached)
|
||||
class TestApp(FlaskTestCase):
|
||||
debug = False
|
||||
|
||||
|
@ -25,7 +25,7 @@ class TestApp(FlaskTestCase):
|
|||
mock_memcached.flush_all()
|
||||
|
||||
def create_app(self):
|
||||
from app.main import application
|
||||
from recommendation.main import application
|
||||
app = application
|
||||
app.config['DEBUG'] = self.debug
|
||||
app.config['PRESERVE_CONTEXT_ON_EXCEPTION'] = False
|
||||
|
@ -55,21 +55,21 @@ class TestApp(FlaskTestCase):
|
|||
def test_no_query(self):
|
||||
eq_(self._get('/').status_code, 400)
|
||||
|
||||
@patch('app.tasks.task_recommend.memcached.get')
|
||||
@patch('recommendation.tasks.task_recommend.memcached.get')
|
||||
def test_exception(self, mock_get):
|
||||
mock_get.side_effect = EXCEPTION
|
||||
response = self._query(QUERY)
|
||||
eq_(response.status_code, 500)
|
||||
eq_(response.json, {})
|
||||
|
||||
@patch('app.tasks.task_recommend.memcached.get')
|
||||
@patch('recommendation.tasks.task_recommend.memcached.get')
|
||||
def test_cache_hit(self, mock_get):
|
||||
mock_get.return_value = RESULTS
|
||||
eq_(self._query(QUERY).status_code, 200)
|
||||
eq_(self._query(QUERY).json, RESULTS)
|
||||
|
||||
@patch('app.tasks.task_recommend.memcached.get')
|
||||
@patch('app.tasks.task_recommend.recommend.delay')
|
||||
@patch('recommendation.tasks.task_recommend.memcached.get')
|
||||
@patch('recommendation.tasks.task_recommend.recommend.delay')
|
||||
def test_cache_miss(self, mock_delay, mock_get):
|
||||
mock_get.return_value = None
|
||||
eq_(self._query(QUERY).status_code, 202)
|
||||
|
@ -79,8 +79,8 @@ class TestApp(FlaskTestCase):
|
|||
class TestAppDebug(TestApp):
|
||||
debug = True
|
||||
|
||||
@patch('app.tasks.task_recommend.memcached.get')
|
||||
@patch('app.tasks.task_recommend.recommend.delay')
|
||||
@patch('recommendation.tasks.task_recommend.memcached.get')
|
||||
@patch('recommendation.tasks.task_recommend.recommend.delay')
|
||||
def test_exception(self, mock_delay, mock_get):
|
||||
mock_get.side_effect = EXCEPTION
|
||||
response = self._query(QUERY)
|
|
@ -5,8 +5,8 @@ from mock import patch
|
|||
from nose.tools import eq_, ok_
|
||||
from wrapt import ObjectProxy
|
||||
|
||||
from app.memorize import memorize, MemorizedObject
|
||||
from app.tests.memcached import mock_memcached
|
||||
from recommendation.memorize import memorize, MemorizedObject
|
||||
from recommendation.tests.memcached import mock_memcached
|
||||
|
||||
|
||||
PREFIX = 'my_prefix'
|
||||
|
@ -58,7 +58,7 @@ class TestMemorizedObject(TestCase):
|
|||
self.instance.cache_key = True
|
||||
|
||||
|
||||
@patch('app.memorize.memcached', mock_memcached)
|
||||
@patch('recommendation.memorize.memcached', mock_memcached)
|
||||
class TestMemorize(TestCase):
|
||||
def setUp(self):
|
||||
self.obj = SampleObject()
|
|
@ -0,0 +1,24 @@
|
|||
from unittest import TestCase
|
||||
|
||||
from flask.app import Flask
|
||||
from nose.tools import eq_, ok_
|
||||
|
||||
|
||||
class TestCeleryApp(TestCase):
|
||||
def test_wsgi_app(self):
|
||||
from recommendation import wsgi as wsgi_module
|
||||
ok_(hasattr(wsgi_module, 'application'))
|
||||
ok_(hasattr(wsgi_module, 'celery'))
|
||||
|
||||
def test_wsgi_app_type(self):
|
||||
from recommendation.wsgi import application
|
||||
eq_(type(application), Flask)
|
||||
|
||||
def test_wsgi_app_singleton(self):
|
||||
"""
|
||||
Tests that multiple imports of the application singleton have the same
|
||||
memory ID (i.e. is the same object).
|
||||
"""
|
||||
from recommendation.wsgi import application as app1
|
||||
from recommendation.wsgi import application as app2
|
||||
eq_(id(app1), id(app2))
|
|
@ -1,6 +1,6 @@
|
|||
from flask.ext.testing import TestCase as FlaskTestCase
|
||||
|
||||
from app.factory import create_app
|
||||
from recommendation.factory import create_app
|
||||
|
||||
|
||||
class AppTestCase(FlaskTestCase):
|
|
@ -2,8 +2,8 @@ import hashlib
|
|||
|
||||
from flask import abort, current_app, Blueprint, jsonify, request
|
||||
|
||||
from app import conf
|
||||
from app.memcached import memcached
|
||||
from recommendation import conf
|
||||
from recommendation.memcached import memcached
|
||||
|
||||
|
||||
main = Blueprint('main', __name__)
|
||||
|
@ -11,7 +11,7 @@ main = Blueprint('main', __name__)
|
|||
|
||||
@main.route('/')
|
||||
def view():
|
||||
from app.tasks.task_recommend import recommend
|
||||
from recommendation.tasks.task_recommend import recommend
|
||||
query = request.args.get('q')
|
||||
if not query:
|
||||
abort(400)
|
|
@ -0,0 +1,11 @@
|
|||
from os import path, pardir
|
||||
from flask import Blueprint, send_file
|
||||
|
||||
|
||||
static = Blueprint('static', __name__)
|
||||
STATIC_DIR = path.join(path.dirname(path.abspath(__file__)), pardir, 'static')
|
||||
|
||||
|
||||
@static.route('/contribute.json')
|
||||
def lbheartbeat():
|
||||
return send_file(path.join(STATIC_DIR, 'contribute.json'))
|
|
@ -2,7 +2,7 @@ from celery.app.control import Control
|
|||
from flask import abort, Blueprint
|
||||
from redis.exceptions import ConnectionError as RedisConnectionError
|
||||
|
||||
from app.memcached import memcached
|
||||
from recommendation.memcached import memcached
|
||||
|
||||
|
||||
status = Blueprint('status', __name__)
|
||||
|
@ -32,7 +32,7 @@ def redis_status():
|
|||
Since our application should not have access to the Redis server, we test
|
||||
this by instantiating a Celery Control and attempting to ping it.
|
||||
"""
|
||||
from app.factory import create_queue
|
||||
from recommendation.factory import create_queue
|
||||
try:
|
||||
Control(app=create_queue()).ping(timeout=1)
|
||||
except RedisConnectionError:
|
||||
|
@ -44,7 +44,7 @@ def celery_status():
|
|||
Raises ServiceDown if any Celery worker servers are down, if any clusters
|
||||
have no workers, or if any workers are down.
|
||||
"""
|
||||
from app.factory import create_queue
|
||||
from recommendation.factory import create_queue
|
||||
clusters = Control(app=create_queue()).ping(timeout=1)
|
||||
if not clusters:
|
||||
raise ServiceDown()
|
|
@ -3,9 +3,9 @@ from urllib.parse import urlencode
|
|||
from mock import patch
|
||||
from nose.tools import eq_, ok_
|
||||
|
||||
from app.cors import cors_headers
|
||||
from app.tests.memcached import mock_memcached
|
||||
from app.tests.util import AppTestCase
|
||||
from recommendation.cors import cors_headers
|
||||
from recommendation.tests.memcached import mock_memcached
|
||||
from recommendation.tests.util import AppTestCase
|
||||
|
||||
|
||||
KEY = 'query_key'
|
||||
|
@ -17,7 +17,7 @@ EXCEPTION_ARGS = ['args']
|
|||
EXCEPTION = RuntimeError(*EXCEPTION_ARGS)
|
||||
|
||||
|
||||
@patch('app.tasks.task_recommend.memcached', mock_memcached)
|
||||
@patch('recommendation.tasks.task_recommend.memcached', mock_memcached)
|
||||
class TestMain(AppTestCase):
|
||||
debug = False
|
||||
|
||||
|
@ -48,21 +48,21 @@ class TestMain(AppTestCase):
|
|||
def test_no_query(self):
|
||||
eq_(self._get('/').status_code, 400)
|
||||
|
||||
@patch('app.tasks.task_recommend.memcached.get')
|
||||
@patch('recommendation.tasks.task_recommend.memcached.get')
|
||||
def test_exception(self, mock_get):
|
||||
mock_get.side_effect = EXCEPTION
|
||||
response = self._query(QUERY)
|
||||
eq_(response.status_code, 500)
|
||||
eq_(response.json, {})
|
||||
|
||||
@patch('app.tasks.task_recommend.memcached.get')
|
||||
@patch('recommendation.tasks.task_recommend.memcached.get')
|
||||
def test_cache_hit(self, mock_get):
|
||||
mock_get.return_value = RESULTS
|
||||
eq_(self._query(QUERY).status_code, 200)
|
||||
eq_(self._query(QUERY).json, RESULTS)
|
||||
|
||||
@patch('app.tasks.task_recommend.memcached.get')
|
||||
@patch('app.tasks.task_recommend.recommend.delay')
|
||||
@patch('recommendation.tasks.task_recommend.memcached.get')
|
||||
@patch('recommendation.tasks.task_recommend.recommend.delay')
|
||||
def test_cache_miss(self, mock_delay, mock_get):
|
||||
mock_get.return_value = None
|
||||
eq_(self._query(QUERY).status_code, 202)
|
||||
|
@ -72,8 +72,8 @@ class TestMain(AppTestCase):
|
|||
class TestMainDebug(TestMain):
|
||||
debug = True
|
||||
|
||||
@patch('app.tasks.task_recommend.memcached.get')
|
||||
@patch('app.tasks.task_recommend.recommend.delay')
|
||||
@patch('recommendation.tasks.task_recommend.memcached.get')
|
||||
@patch('recommendation.tasks.task_recommend.recommend.delay')
|
||||
def test_exception(self, mock_delay, mock_get):
|
||||
mock_get.side_effect = EXCEPTION
|
||||
response = self._query(QUERY)
|
|
@ -0,0 +1,15 @@
|
|||
import json
|
||||
from os import path
|
||||
|
||||
from nose.tools import eq_
|
||||
|
||||
from recommendation.tests.util import AppTestCase
|
||||
from recommendation.views.static import STATIC_DIR
|
||||
|
||||
|
||||
class TestStaticViews(AppTestCase):
|
||||
def test_contribute(self):
|
||||
response = self.client.get('/contribute.json')
|
||||
eq_(response.status_code, 200)
|
||||
with open(path.join(STATIC_DIR, 'contribute.json')) as file_data:
|
||||
eq_(json.load(file_data), response.json)
|
|
@ -2,9 +2,10 @@ from mock import patch
|
|||
from nose.tools import eq_, ok_
|
||||
from redis.exceptions import ConnectionError as RedisError
|
||||
|
||||
from app.views.status import (celery_status, memcached_status, redis_status,
|
||||
ServiceDown)
|
||||
from app.tests.util import AppTestCase
|
||||
from recommendation.views.status import (celery_status, memcached_status,
|
||||
redis_status, ServiceDown)
|
||||
from recommendation.tests.util import AppTestCase
|
||||
|
||||
|
||||
MEMCACHED_WORKER_BAD = {'not ok': 'not pong'}
|
||||
MEMCACHED_WORKER_OK = {'ok': 'pong'}
|
||||
|
@ -32,9 +33,9 @@ class TestStatusViews(AppTestCase):
|
|||
eq_(response.status_code, 200)
|
||||
ok_(not response.data)
|
||||
|
||||
@patch('app.views.status.celery_status')
|
||||
@patch('app.views.status.memcached_status')
|
||||
@patch('app.views.status.redis_status')
|
||||
@patch('recommendation.views.status.celery_status')
|
||||
@patch('recommendation.views.status.memcached_status')
|
||||
@patch('recommendation.views.status.redis_status')
|
||||
def test_heartbeat_pass(self, mock_celery, mock_memcached, mock_redis):
|
||||
response = self.client.get('/__heartbeat__')
|
||||
eq_(response.status_code, 200)
|
||||
|
@ -42,18 +43,18 @@ class TestStatusViews(AppTestCase):
|
|||
eq_(mock_memcached.call_count, 1)
|
||||
eq_(mock_redis.call_count, 1)
|
||||
|
||||
@patch('app.views.status.celery_status')
|
||||
@patch('app.views.status.memcached_status')
|
||||
@patch('app.views.status.redis_status')
|
||||
@patch('recommendation.views.status.celery_status')
|
||||
@patch('recommendation.views.status.memcached_status')
|
||||
@patch('recommendation.views.status.redis_status')
|
||||
def test_heartbeat_celery(self, mock_celery, mock_memcached, mock_redis):
|
||||
mock_celery.side_effect = ServiceDown
|
||||
response = self.client.get('/__heartbeat__')
|
||||
eq_(response.status_code, 500)
|
||||
eq_(mock_celery.call_count, 1)
|
||||
|
||||
@patch('app.views.status.celery_status')
|
||||
@patch('app.views.status.memcached_status')
|
||||
@patch('app.views.status.redis_status')
|
||||
@patch('recommendation.views.status.celery_status')
|
||||
@patch('recommendation.views.status.memcached_status')
|
||||
@patch('recommendation.views.status.redis_status')
|
||||
def test_heartbeat_memcached(self, mock_celery, mock_memcached,
|
||||
mock_redis):
|
||||
mock_memcached.side_effect = ServiceDown
|
||||
|
@ -61,57 +62,57 @@ class TestStatusViews(AppTestCase):
|
|||
eq_(response.status_code, 500)
|
||||
eq_(mock_memcached.call_count, 1)
|
||||
|
||||
@patch('app.views.status.celery_status')
|
||||
@patch('app.views.status.memcached_status')
|
||||
@patch('app.views.status.redis_status')
|
||||
@patch('recommendation.views.status.celery_status')
|
||||
@patch('recommendation.views.status.memcached_status')
|
||||
@patch('recommendation.views.status.redis_status')
|
||||
def test_heartbeat_redis(self, mock_celery, mock_memcached, mock_redis):
|
||||
mock_redis.side_effect = ServiceDown
|
||||
response = self.client.get('/__heartbeat__')
|
||||
eq_(response.status_code, 500)
|
||||
eq_(mock_redis.call_count, 1)
|
||||
|
||||
@patch('app.views.status.memcached.set')
|
||||
@patch('recommendation.views.status.memcached.set')
|
||||
def test_memcached_status_pass(self, mock_set):
|
||||
mock_set.return_value = True
|
||||
memcached_status()
|
||||
self.assert_(True)
|
||||
|
||||
@patch('app.views.status.memcached.set')
|
||||
@patch('recommendation.views.status.memcached.set')
|
||||
def test_memcached_status_fail(self, mock_set):
|
||||
mock_set.return_value = 0
|
||||
with self.assertRaises(ServiceDown):
|
||||
memcached_status()
|
||||
|
||||
@patch('app.views.status.Control.ping')
|
||||
@patch('recommendation.views.status.Control.ping')
|
||||
def test_redis_status_pass(self, mock_ping):
|
||||
redis_status()
|
||||
self.assert_(True)
|
||||
|
||||
@patch('app.views.status.Control.ping')
|
||||
@patch('recommendation.views.status.Control.ping')
|
||||
def test_redis_status_fail(self, mock_ping):
|
||||
mock_ping.side_effect = RedisError
|
||||
with self.assertRaises(ServiceDown):
|
||||
redis_status()
|
||||
|
||||
@patch('app.views.status.Control.ping')
|
||||
@patch('recommendation.views.status.Control.ping')
|
||||
def test_celery_status_pass(self, mock_ping):
|
||||
mock_ping.return_value = MEMCACHED_PING_OK
|
||||
celery_status()
|
||||
self.assert_(True)
|
||||
|
||||
@patch('app.views.status.Control.ping')
|
||||
@patch('recommendation.views.status.Control.ping')
|
||||
def test_celery_status_no_workers(self, mock_ping):
|
||||
mock_ping.return_value = MEMCACHED_PING_NO_WORKERS
|
||||
with self.assertRaises(ServiceDown):
|
||||
celery_status()
|
||||
|
||||
@patch('app.views.status.Control.ping')
|
||||
@patch('recommendation.views.status.Control.ping')
|
||||
def test_celery_status_no_clusters(self, mock_ping):
|
||||
mock_ping.return_value = MEMCACHED_PING_NO_CLUSTERS
|
||||
with self.assertRaises(ServiceDown):
|
||||
celery_status()
|
||||
|
||||
@patch('app.views.status.Control.ping')
|
||||
@patch('recommendation.views.status.Control.ping')
|
||||
def test_celery_status_workers_down(self, mock_ping):
|
||||
mock_ping.return_value = MEMCACHED_PING_BAD
|
||||
with self.assertRaises(ServiceDown):
|
|
@ -0,0 +1,4 @@
|
|||
from recommendation.celery import celery # noqa
|
||||
from recommendation.factory import create_app
|
||||
|
||||
application = create_app()
|
|
@ -12,6 +12,7 @@ Jinja2==2.8
|
|||
MarkupSafe==0.23
|
||||
mock==1.3.0
|
||||
nose==1.3.7
|
||||
nose-exclude==0.4.1
|
||||
oauth2==1.9.0.post1
|
||||
redis==2.10.5
|
||||
requests==2.9.1
|
12
server.py
12
server.py
|
@ -1,12 +0,0 @@
|
|||
import os
|
||||
import sys
|
||||
|
||||
from app import conf
|
||||
from app.main import app
|
||||
|
||||
|
||||
sys.path.append(os.path.dirname(__file__))
|
||||
|
||||
if __name__ == '__main__':
|
||||
print('Running server on http://%s:%d' % (conf.HOST, conf.PORT))
|
||||
app.run(debug=conf.DEBUG, host=conf.HOST, port=conf.PORT)
|
|
@ -1,2 +1,2 @@
|
|||
[aliases]
|
||||
test=nosetests --with-coverage --cover-package=app --cover-min-percentage=100 --exclude=app/src
|
||||
test=nosetests --with-coverage --cover-package=recommendation --cover-min-percentage=100 --exclude=recommendation/src
|
||||
|
|
Загрузка…
Ссылка в новой задаче