Merge pull request #52 from chuckharmston/51-single-container

Refactors into single-container application with supplementary services in compose (closes #51).
This commit is contained in:
Chuck Harmston 2016-02-26 12:50:46 -07:00
Родитель 36becf6841 132538f734
Коммит e1b6e759d8
80 изменённых файлов: 395 добавлений и 295 удалений

Просмотреть файл

@ -1,2 +1,5 @@
[run]
source = app
[report]
omit = recommendation/conf.py

9
.dockerignore Normal file
Просмотреть файл

@ -0,0 +1,9 @@
__pycache__
*.pyc
src
.DS_Store
.eggs
.git
dist
.coverage
coverage.xml

Просмотреть файл

@ -2,7 +2,7 @@ language: python
python:
- "3.5"
install:
- "pip install -r app/requirements.txt"
- "pip install -r requirements.txt"
script:
- "nosetests --cover-min-percentage=100 --with-coverage --cover-package=app"
- "nosetests --with-coverage --cover-package=recommendation --exclude-dir=src"
after_success: coveralls

19
Dockerfile Normal file
Просмотреть файл

@ -0,0 +1,19 @@
# universal-search-recommendation
#
# VERSION 0.1
FROM python:3.5
MAINTAINER https://mail.mozilla.org/listinfo/testpilot-dev
RUN groupadd --gid 10001 app && \
useradd --uid 10001 --gid 10001 --shell /usr/sbin/nologin app
COPY ./requirements.txt /app/requirements.txt
RUN pip install --upgrade --no-cache-dir -r /app/requirements.txt
COPY . /app
ENV PYTHONPATH $PYTHONPATH:/app
USER app
EXPOSE 8000
ENTRYPOINT ["/app/conf/web.sh"]

Просмотреть файл

Просмотреть файл

@ -1,3 +0,0 @@
__pycache__
src
.DS_Store

Просмотреть файл

@ -1,19 +0,0 @@
# universal-search-recommendation-app
#
# VERSION 0.1
FROM python:3.5
MAINTAINER https://mail.mozilla.org/listinfo/testpilot-dev
RUN groupadd --gid 1001 app && \
useradd --uid 1001 --gid 1001 --shell /usr/sbin/nologin app
WORKDIR /app
COPY ./requirements.txt /app/requirements.txt
RUN pip install --upgrade --no-cache-dir -r requirements.txt
COPY . /app
ENV PYTHONPATH $PYTHONPATH:/:/app
USER app

Просмотреть файл

@ -1,17 +0,0 @@
from os import environ as env
CELERY_BROKER_URL = env.get('CELERY_BROKER_URL', 'redis://redis:6379/0')
DEBUG = env.get('RECOMMENDATION_ENV', 'development') == 'development'
KEY_PREFIX = env.get('RECOMMENDATION_KEY_PREFIX', 'query_')
MEMCACHED_HOST = env.get('MEMCACHED_HOST', 'memcached:11211')
EMBEDLY_API_KEY = env.get('EMBEDLY_API_KEY', '')
# Yahoo BOSS API credentials.
YAHOO_OAUTH_KEY = env.get('YAHOO_OAUTH_KEY', '')
YAHOO_OAUTH_SECRET = env.get('YAHOO_OAUTH_SECRET', '')
# For non-Docker development server.
HOST = env.get('RECOMMENDATION_HOST', '0.0.0.0')
PORT = env.get('RECOMMENDATION_PORT', 5000)

Просмотреть файл

@ -1,3 +0,0 @@
#!/usr/bin/env bash
uwsgi --http :${PORT:-8000} --wsgi-file /app/main.py --master

Просмотреть файл

@ -1,5 +0,0 @@
from app import tasks # noqa
from app.factory import create_app, create_queue
application = create_app()
celery = create_queue()

Просмотреть файл

@ -1,10 +0,0 @@
from app.search.classification.domain import DomainClassifier
from app.search.classification.embedly import EmbedlyClassifier
from app.search.classification.wikipedia import WikipediaClassifier
CLASSIFIERS = [
DomainClassifier,
EmbedlyClassifier,
WikipediaClassifier,
]

Просмотреть файл

@ -1 +0,0 @@
from app.tasks.task_recommend import recommend # noqa

Просмотреть файл

@ -18,7 +18,7 @@ dependencies:
- printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s"}\n' "$CIRCLE_SHA1" "$CIRCLE_TAG" "$CIRCLE_PROJECT_USERNAME" "$CIRCLE_PROJECT_REPONAME" > version.json
# build the actual deployment container
- docker build -t app:build app
- docker build -t app:build .
# Clean up any old images; save the new one.
- I="image-$(date +%j).tgz"; mkdir -p ~/docker; rm ~/docker/*; docker save app:build | gzip -c > ~/docker/$I; ls -l ~/docker
@ -27,7 +27,7 @@ dependencies:
# Run flake8 and the Python test suite via nose.
test:
override:
- docker run app:build sh -c 'cd / && flake8 /app --exclude=/app/src && nosetests --exclude=/app/src'
- docker run -d app:build sh -c 'flake8 . --exclude=app/src && nosetests --exclude-dir=app/src'
# Tag and push the container to Docker Hub.

Просмотреть файл

@ -1 +0,0 @@
.DS_Store

Просмотреть файл

@ -1,12 +0,0 @@
# universal-search-recommendation-proxy
#
# VERSION 0.1
FROM nginx:1.9.9
MAINTAINER https://mail.mozilla.org/listinfo/testpilot-dev
RUN rm /etc/nginx/conf.d/default.conf
COPY recommendation.conf /etc/nginx/conf.d/
RUN mkdir -p /var/www/
COPY contribute.json /var/www/

Просмотреть файл

@ -1,16 +0,0 @@
server {
listen 80;
server_name "";
charset utf-8;
location /contribute.json {
alias /var/www/contribute.json;
}
location / {
proxy_pass http://app:8000;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
}

3
conf/web.sh Executable file
Просмотреть файл

@ -0,0 +1,3 @@
#!/usr/bin/env bash
uwsgi --http :${PORT:-8000} --wsgi-file /app/recommendation/wsgi.py --master

3
conf/worker.sh Executable file
Просмотреть файл

@ -0,0 +1,3 @@
#!/usr/bin/env bash
celery worker --app=recommendation

Просмотреть файл

@ -1,37 +1,26 @@
app:
build: ./app
command: uwsgi --http :8000 --wsgi-file /app/main.py --master
env_file: .env
expose:
- "8000"
links:
- memcached
- redis
restart: always
nginx:
build: ./conf/nginx
links:
- app
ports:
- "80:80"
restart: always
memcached:
image: memcached
restart: always
expose:
- "11211"
image: memcached
restart: always
ports:
- "11211"
- "11211:11211"
redis:
expose:
- "6379"
image: redis
ports:
- "6379"
- "6379:6379"
celery:
build: ./app
command: celery worker --app=app.main
worker:
build: ./
entrypoint: /app/conf/worker.sh
env_file: .env
environment:
- CELERY_BROKER_URL=redis://redis:6379/0
- RECOMMENDATION_ENV="worker"
links:
- memcached
- redis

Просмотреть файл

@ -1,40 +1,86 @@
# Local Development
## Local Development
A multiple-container Docker configuration is maintained to support local development.
## First-time setup
### Configuration
A few steps are required the first time
Some environment variables are required. Each of those are contained within `.env.dist`, which should be copied to `.env` and populated as appropriate.
### Services
`universal-search-recommendation` requires several additional services: a [Memcached](http://memcached.org/) cache backend, a [Celery](http://www.celeryproject.org/) task queue, and a [Redis](http://redis.io/) Celery backend. A [Compose](https://docs.docker.com/compose/) configuration is included to streamline setup and management of those services.
#### Set up Docker host.
The first time you set up the services, you must create the Docker host.
- [Install Docker Toolbox](https://www.docker.com/products/docker-toolbox).
- Some environment variables are required. Each of those are contained within `.env.dist`, which should be copied to `.env` and populated as appropriate.
- Create a Docker host:
```bash
docker-machine create --driver virtualbox universal-search
docker-machine create --driver virtualbox universal-search-dev
```
- Add an entry to your hosts file:
- Add an entry to your `hosts` file:
```bash
sudo sh -c "echo $(docker-machine ip universal-search 2>/dev/null) universal-search.dev >> /etc/hosts"
sudo sh -c "echo $(docker-machine ip universal-search-dev 2>/dev/null) universal-search.dev >> /etc/hosts"
```
## Start machine and containers
#### Start services.
The machine and containers can then be started at any time:
After the host is set up, you can run each service by:
```
docker-machine start universal-search-dev
eval $(docker-machine env universal-search-dev)
docker-compose up -d
```
To verify that each service is up, run:
```bash
docker-machine start universal-search
eval $(docker-machine env universal-search)
docker-compose up
docker-compose ps
```
While `docker-compose up` is running, the recommendation server may be accessed at [http://universal-search.dev](http://universal-search.dev).
In the output you should see containers named `recommendation_memcached_1`, `recommendation_redis_1`, and `recommendation_worker_1`. The state for each of these should be `Up`.
## Architecture
### Application
An nginx server is the entry point, acting as a reverse proxy to forward requests to a Python container running uWSGI, which manages the application server. Memcached, Celery, and Redis containers run parallel; used for caching, as a task queue, and as a backend for the task queue, respectively.
`universal-search-recommendation` is managed as a Docker container that is separately built from the services, but run on the same host as the services.
To build it:
```bash
docker build -t universal-search-recommendation .
```
To run it:
```bash
docker run -d -e "RECOMMENDATION_SERVICES=`docker-machine ip universal-search-dev`" -p 80:8000/tcp universal-search-recommendation
```
To verify that the application is running:
```bash
docker ps --filter ancestor="universal-search-recommendation"
```
You should see one container running, and its status should begin with `Up`.
### Usage
If the application and services are all running, you should be able to access to access the recommendation server at [http://universal-search.dev](http://universal-search.dev). After making changes to the application, you will need to stop it:
```bash
docker stop $(docker ps --filter ancestor="universal-search-recommendation" --format="{{.ID}}")
```
Then rebuild and start it back up, per above.

Просмотреть файл

4
recommendation/celery.py Normal file
Просмотреть файл

@ -0,0 +1,4 @@
from recommendation import tasks # noqa
from recommendation.factory import create_queue
celery = create_queue()

17
recommendation/conf.py Normal file
Просмотреть файл

@ -0,0 +1,17 @@
from os import environ as env
DEBUG = env.get('RECOMMENDATION_ENV', 'development') == 'development'
KEY_PREFIX = env.get('RECOMMENDATION_KEY_PREFIX', 'query_')
EMBEDLY_API_KEY = env.get('EMBEDLY_API_KEY', '')
YAHOO_OAUTH_KEY = env.get('YAHOO_OAUTH_KEY', '')
YAHOO_OAUTH_SECRET = env.get('YAHOO_OAUTH_SECRET', '')
RECOMMENDATION_SERVICES = env.get('RECOMMENDATION_SERVICES')
if DEBUG and RECOMMENDATION_SERVICES:
CELERY_BROKER_URL = 'redis://%s:6379/0' % RECOMMENDATION_SERVICES
MEMCACHED_HOST = '%s:11211' % RECOMMENDATION_SERVICES
else:
CELERY_BROKER_URL = env.get('CELERY_BROKER_URL', 'redis://redis:6379/0')
MEMCACHED_HOST = env.get('MEMCACHED_HOST', 'memcached:11211')

Просмотреть файл

Просмотреть файл

@ -1,16 +1,18 @@
from celery import Celery
from flask import Flask
from app import conf
from app.cors import cors_headers
from app.views.main import main
from app.views.status import status
from recommendation import conf
from recommendation.cors import cors_headers
from recommendation.views.main import main
from recommendation.views.static import static
from recommendation.views.status import status
def create_app():
app = Flask(__name__)
app.after_request(cors_headers)
app.register_blueprint(main)
app.register_blueprint(static)
app.register_blueprint(status)
app.config.update(
CELERY_BROKER_URL=conf.CELERY_BROKER_URL,

5
recommendation/main.py Normal file
Просмотреть файл

@ -0,0 +1,5 @@
from recommendation import tasks # noqa
from recommendation.factory import create_app, create_queue
application = create_app()
celery = create_queue()

Просмотреть файл

@ -1,6 +1,6 @@
from memcache import Client
from app import conf
from recommendation import conf
memcached = Client([conf.MEMCACHED_HOST])

Просмотреть файл

@ -5,7 +5,7 @@ from inspect import getargspec
from wrapt import ObjectProxy
from app.memcached import memcached
from recommendation.memcached import memcached
class MemorizedObject(ObjectProxy):

Просмотреть файл

@ -0,0 +1,10 @@
from recommendation.search.classification.domain import DomainClassifier
from recommendation.search.classification.embedly import EmbedlyClassifier
from recommendation.search.classification.wikipedia import WikipediaClassifier
CLASSIFIERS = [
DomainClassifier,
EmbedlyClassifier,
WikipediaClassifier,
]

Просмотреть файл

@ -1,4 +1,4 @@
from app.search.classification.base import BaseClassifier
from recommendation.search.classification.base import BaseClassifier
class DomainClassifier(BaseClassifier):

Просмотреть файл

@ -2,11 +2,10 @@ from urllib.parse import urlencode
import requests
import app.conf
from app.memorize import memorize
from app.search.classification.base import BaseClassifier
from app.search.classification.wikipedia import WikipediaClassifier
from recommendation import conf
from recommendation.memorize import memorize
from recommendation.search.classification.base import BaseClassifier
from recommendation.search.classification.wikipedia import WikipediaClassifier
class EmbedlyClassifier(BaseClassifier):
@ -51,7 +50,7 @@ class EmbedlyClassifier(BaseClassifier):
def _api_url(self, url):
return '%s?%s' % (self.api_url, urlencode({
'key': app.conf.EMBEDLY_API_KEY,
'key': conf.EMBEDLY_API_KEY,
'words': 20,
'secure': 'true',
'url': url

Просмотреть файл

@ -1,6 +1,6 @@
from unittest import TestCase
from app.search.classification.base import BaseClassifier
from recommendation.search.classification.base import BaseClassifier
class TestBaseClassifier(TestCase):

Просмотреть файл

@ -3,7 +3,7 @@ from urllib.parse import ParseResult
from nose.tools import eq_, ok_
from app.search.classification.domain import DomainClassifier
from recommendation.search.classification.domain import DomainClassifier
DOMAIN = 'www.mozilla.com'

Просмотреть файл

@ -5,8 +5,8 @@ from urllib.parse import parse_qs, urlparse
import responses
from nose.tools import eq_, ok_
from app.search.classification.embedly import EmbedlyClassifier
from app.tests.memcached import mock_memcached
from recommendation.search.classification.embedly import EmbedlyClassifier
from recommendation.tests.memcached import mock_memcached
MOCK_API_KEY = '0123456789abcdef'
@ -92,7 +92,7 @@ class TestEmbedlyClassifier(TestCase):
eq_(self._matches('https://en.wikipedia.org/wiki/Mozilla'), True)
eq_(self._matches('https://en.wikipedia.org/wiki/Mozilla/'), True)
@patch('app.conf.EMBEDLY_API_KEY', MOCK_API_KEY)
@patch('recommendation.conf.EMBEDLY_API_KEY', MOCK_API_KEY)
def test_api_url(self):
api_url = self._api_url(MOCK_RESULT_URL)
api_qs = parse_qs(urlparse(api_url).query)
@ -101,8 +101,9 @@ class TestEmbedlyClassifier(TestCase):
eq_(api_qs['url'][0], MOCK_RESULT_URL)
eq_(api_qs['secure'][0], 'true')
@patch('app.memorize.memcached', mock_memcached)
@patch('app.search.classification.embedly.EmbedlyClassifier._api_url')
@patch('recommendation.memorize.memcached', mock_memcached)
@patch('recommendation.search.classification.embedly.EmbedlyClassifier'
'._api_url')
@responses.activate
def test_api_response(self, mock_api_url):
mock_api_url.return_value = MOCK_API_URL
@ -116,8 +117,9 @@ class TestEmbedlyClassifier(TestCase):
eq_(response_cold.cache_key, response_warm.cache_key)
eq_(response_cold, response_warm, MOCK_RESPONSE)
@patch('app.memorize.memcached', mock_memcached)
@patch('app.search.classification.embedly.EmbedlyClassifier._api_url')
@patch('recommendation.memorize.memcached', mock_memcached)
@patch('recommendation.search.classification.embedly.EmbedlyClassifier'
'._api_url')
@responses.activate
def test_enhance(self, mock_api_url):
mock_api_url.return_value = MOCK_API_URL
@ -130,8 +132,10 @@ class TestEmbedlyClassifier(TestCase):
eq_(enhanced['favicon']['colors'], MOCK_RESPONSE['favicon_colors'])
eq_(enhanced['favicon']['url'], MOCK_RESPONSE['favicon_url'])
@patch('app.search.classification.embedly.EmbedlyClassifier._api_response')
@patch('app.search.classification.embedly.EmbedlyClassifier._get_image')
@patch('recommendation.search.classification.embedly.EmbedlyClassifier'
'._api_response')
@patch('recommendation.search.classification.embedly.EmbedlyClassifier'
'._get_image')
def test_enhance_no_images(self, mock_get_image, mock_api_response):
mock_api_response.return_value = MOCK_RESPONSE
mock_get_image.side_effect = KeyError

Просмотреть файл

@ -5,8 +5,8 @@ from urllib.parse import parse_qs, urlparse
import responses
from nose.tools import eq_, ok_
from app.search.classification.wikipedia import WikipediaClassifier
from app.tests.memcached import mock_memcached
from recommendation.search.classification.wikipedia import WikipediaClassifier
from recommendation.tests.memcached import mock_memcached
SLUG = 'Mozilla'
@ -70,8 +70,9 @@ class TestWikipediaClassifier(TestCase):
actual[4] = parse_qs(actual[4])
eq_(expected, actual)
@patch('app.memorize.memcached', mock_memcached)
@patch('app.search.classification.wikipedia.WikipediaClassifier._api_url')
@patch('recommendation.memorize.memcached', mock_memcached)
@patch('recommendation.search.classification.wikipedia.WikipediaClassifier'
'._api_url')
@responses.activate
def test_api_response(self, mock_api_url):
"""
@ -94,8 +95,9 @@ class TestWikipediaClassifier(TestCase):
eq_(response_cold.cache_key, response_warm.cache_key)
eq_(response_cold, response_warm, EXPECTED_SANITIZED_RESPONSE)
@patch('app.memorize.memcached', mock_memcached)
@patch('app.search.classification.wikipedia.WikipediaClassifier._api_url')
@patch('recommendation.memorize.memcached', mock_memcached)
@patch('recommendation.search.classification.wikipedia.WikipediaClassifier'
'._api_url')
@responses.activate
def test_enhance(self, mock_api_url):
"""

Просмотреть файл

@ -2,8 +2,8 @@ from urllib.parse import urlencode, urlunparse
import requests
from app.memorize import memorize
from app.search.classification.base import BaseClassifier
from recommendation.memorize import memorize
from recommendation.search.classification.base import BaseClassifier
class WikipediaClassifier(BaseClassifier):

Просмотреть файл

@ -1,6 +1,6 @@
import bleach
from app.memorize import memorize
from recommendation.memorize import memorize
class BaseQueryEngine(object):

Просмотреть файл

@ -3,8 +3,8 @@ from unittest.mock import patch
from nose.tools import eq_, ok_
from app.search.query.base import BaseQueryEngine
from app.tests.memcached import mock_memcached
from recommendation.search.query.base import BaseQueryEngine
from recommendation.tests.memcached import mock_memcached
QUERY = 'loveland'
@ -41,9 +41,10 @@ class TestBaseQueryEngine(TestCase):
def test_get_best_result(self):
eq_(self.instance.get_best_result(MOCK_RESULTS), MOCK_RESULTS[0])
@patch('app.search.query.base.BaseQueryEngine.get_result_abstract')
@patch('app.search.query.base.BaseQueryEngine.get_result_title')
@patch('app.search.query.base.BaseQueryEngine.get_result_url')
@patch('recommendation.search.query.base.BaseQueryEngine'
'.get_result_abstract')
@patch('recommendation.search.query.base.BaseQueryEngine.get_result_title')
@patch('recommendation.search.query.base.BaseQueryEngine.get_result_url')
def test_sanitize_result(self, *args):
result = self.instance.sanitize_result(MOCK_RESULTS[0])
ok_(all(k in result.keys() for k in ['abstract', 'title', 'url']))
@ -69,8 +70,8 @@ class TestBaseQueryEngine(TestCase):
eq_(self.instance.get_result_abstract(MOCK_RESULTS[0]),
MOCK_RESULT_SANITIZED['abstract'])
@patch('app.search.query.base.BaseQueryEngine.fetch')
@patch('app.memorize.memcached', mock_memcached)
@patch('recommendation.search.query.base.BaseQueryEngine.fetch')
@patch('recommendation.memorize.memcached', mock_memcached)
def test_search(self, mock_fetch):
mock_fetch.return_value = MOCK_RESULTS
results_cold = self.instance.search(QUERY)

Просмотреть файл

@ -5,8 +5,8 @@ from urllib.parse import parse_qs, urlparse
import responses
from nose.tools import eq_, ok_
from app.search.query.yahoo import YahooQueryEngine
from app.tests.memcached import mock_memcached
from recommendation.search.query.yahoo import YahooQueryEngine
from recommendation.tests.memcached import mock_memcached
QUERY = 'loveland'
@ -56,7 +56,7 @@ class TestYahooClassifier(TestCase):
ok_(len(query_string['oauth_version']))
eq_(query_string['q'][0], QUERY)
@patch('app.memorize.memcached', mock_memcached)
@patch('recommendation.memorize.memcached', mock_memcached)
@responses.activate
def test_fetch(self):
responses.add(responses.GET, YahooQueryEngine.url,
@ -68,7 +68,7 @@ class TestYahooClassifier(TestCase):
eq_(response_cold.cache_key, response_warm.cache_key)
eq_(response_cold, response_warm, MOCK_RESPONSE)
@patch('app.memorize.memcached', mock_memcached)
@patch('recommendation.memorize.memcached', mock_memcached)
@responses.activate
def test_sanitize_response(self):
responses.add(responses.GET, YahooQueryEngine.url,

Просмотреть файл

@ -4,9 +4,9 @@ from urllib.parse import quote_plus
import oauth2
import requests
from app import conf
from app.memorize import memorize
from app.search.query.base import BaseQueryEngine
from recommendation import conf
from recommendation.memorize import memorize
from recommendation.search.query.base import BaseQueryEngine
class YahooQueryEngine(BaseQueryEngine):

Просмотреть файл

@ -1,6 +1,6 @@
from app.search.classification import CLASSIFIERS
from app.search.query.yahoo import YahooQueryEngine
from app.search.suggest.bing import BingSuggestionEngine
from recommendation.search.classification import CLASSIFIERS
from recommendation.search.query.yahoo import YahooQueryEngine
from recommendation.search.suggest.bing import BingSuggestionEngine
class SearchRecommendation(object):

Просмотреть файл

@ -1,4 +1,4 @@
from app.memorize import memorize
from recommendation.memorize import memorize
class BaseSuggestionEngine(object):

Просмотреть файл

@ -1,7 +1,7 @@
import requests
from app.memorize import memorize
from app.search.suggest.base import BaseSuggestionEngine
from recommendation.memorize import memorize
from recommendation.search.suggest.base import BaseSuggestionEngine
class BingSuggestionEngine(BaseSuggestionEngine):

Просмотреть файл

@ -3,8 +3,8 @@ from unittest import TestCase
from mock import patch
from nose.tools import eq_, ok_
from app.search.suggest.base import BaseSuggestionEngine
from app.tests.memcached import mock_memcached
from recommendation.search.suggest.base import BaseSuggestionEngine
from recommendation.tests.memcached import mock_memcached
QUERY = ''
@ -29,8 +29,8 @@ class TestBaseSuggestionEngine(TestCase):
def test_sanitize(self):
eq_(self.instance.sanitize(RESULTS), RESULTS)
@patch('app.search.suggest.base.BaseSuggestionEngine.fetch')
@patch('app.memorize.memcached', mock_memcached)
@patch('recommendation.search.suggest.base.BaseSuggestionEngine.fetch')
@patch('recommendation.memorize.memcached', mock_memcached)
def test_search(self, mock_fetch):
mock_fetch.return_value = RESULTS
results_cold = self.instance.search(QUERY)

Просмотреть файл

@ -4,8 +4,8 @@ import responses
from mock import patch
from nose.tools import eq_, ok_
from app.search.suggest.bing import BingSuggestionEngine
from app.tests.memcached import mock_memcached
from recommendation.search.suggest.bing import BingSuggestionEngine
from recommendation.tests.memcached import mock_memcached
QUERY = 'original query'
@ -23,7 +23,7 @@ class TestBingSuggestionEngine(TestCase):
def setUp(self):
self.instance = BingSuggestionEngine(QUERY)
@patch('app.memorize.memcached', mock_memcached)
@patch('recommendation.memorize.memcached', mock_memcached)
@responses.activate
def test_fetch(self):
responses.add(responses.GET, BingSuggestionEngine.url, json=RESPONSE,

Просмотреть файл

Просмотреть файл

@ -3,18 +3,18 @@ from unittest import TestCase
from mock import patch
from nose.tools import eq_, ok_
from app.search.classification.domain import DomainClassifier
from app.search.classification.tests.test_domain import DOMAIN
from app.search.recommendation import SearchRecommendation
from app.search.suggest.base import BaseSuggestionEngine
from app.search.suggest.bing import BingSuggestionEngine
from app.search.suggest.tests.test_bing import (QUERY as BING_QUERY,
RESULTS as BING_RESULTS)
from app.search.query.base import BaseQueryEngine
from app.search.query.yahoo import YahooQueryEngine
from app.search.query.tests.test_yahoo import (QUERY as YAHOO_QUERY,
MOCK_RESPONSE as YAHOO_RESPONSE)
from app.tests.memcached import mock_memcached
from recommendation.search.classification.domain import DomainClassifier
from recommendation.search.classification.tests.test_domain import DOMAIN
from recommendation.search.recommendation import SearchRecommendation
from recommendation.search.suggest.base import BaseSuggestionEngine
from recommendation.search.suggest.bing import BingSuggestionEngine
from recommendation.search.suggest.tests.test_bing import (
QUERY as BING_QUERY, RESULTS as BING_RESULTS)
from recommendation.search.query.base import BaseQueryEngine
from recommendation.search.query.yahoo import YahooQueryEngine
from recommendation.search.query.tests.test_yahoo import (
QUERY as YAHOO_QUERY, MOCK_RESPONSE as YAHOO_RESPONSE)
from recommendation.tests.memcached import mock_memcached
QUERY = 'Cubs'
@ -35,7 +35,8 @@ class TestSearchRecommendation(TestCase):
engine = self.instance.get_query_engine()
ok_(issubclass(engine, BaseQueryEngine))
@patch('app.search.classification.domain.DomainClassifier.is_match')
@patch('recommendation.search.classification.domain.DomainClassifier'
'.is_match')
def test_get_classifiers(self, mock_match):
mock_match.return_value = True
classifiers = self.instance.get_classifiers({
@ -45,9 +46,9 @@ class TestSearchRecommendation(TestCase):
ok_(isinstance(classifiers[0], DomainClassifier))
return classifiers
@patch(('app.search.recommendation.SearchRecommendation.get_suggestion_eng'
'ine'))
@patch('app.search.suggest.bing.BingSuggestionEngine.search')
@patch(('recommendation.search.recommendation.SearchRecommendation'
'.get_suggestion_engine'))
@patch('recommendation.search.suggest.bing.BingSuggestionEngine.search')
def test_get_suggestions(self, mock_bing, mock_suggestion_engine):
mock_bing.return_value = BING_RESULTS
mock_suggestion_engine.return_value = BingSuggestionEngine
@ -57,8 +58,9 @@ class TestSearchRecommendation(TestCase):
def test_get_top_suggestion(self):
eq_(self.instance.get_top_suggestion(BING_RESULTS), BING_RESULTS[0])
@patch('app.search.recommendation.SearchRecommendation.get_query_engine')
@patch('app.search.query.yahoo.YahooQueryEngine.search')
@patch('recommendation.search.recommendation.SearchRecommendation'
'.get_query_engine')
@patch('recommendation.search.query.yahoo.YahooQueryEngine.search')
def test_do_query(self, mock_yahoo, mock_query_engine):
response = YAHOO_RESPONSE['bossresponse']['web']['results'][0]
mock_yahoo.return_value = response
@ -66,11 +68,15 @@ class TestSearchRecommendation(TestCase):
eq_(self.instance.do_query(YAHOO_QUERY), response)
eq_(mock_yahoo.call_count, 1)
@patch('app.search.recommendation.SearchRecommendation.get_classifiers')
@patch('app.search.recommendation.SearchRecommendation.do_query')
@patch('app.search.recommendation.SearchRecommendation.get_top_suggestion')
@patch('app.search.recommendation.SearchRecommendation.get_suggestions')
@patch('app.memorize.memcached', mock_memcached)
@patch('recommendation.search.recommendation.SearchRecommendation'
'.get_classifiers')
@patch('recommendation.search.recommendation.SearchRecommendation'
'.do_query')
@patch('recommendation.search.recommendation.SearchRecommendation'
'.get_top_suggestion')
@patch('recommendation.search.recommendation.SearchRecommendation'
'.get_suggestions')
@patch('recommendation.memorize.memcached', mock_memcached)
def test_do_search_get_recommendation(self, mock_suggestions,
mock_top_suggestion, mock_result,
mock_classifiers):

Просмотреть файл

Просмотреть файл

@ -0,0 +1 @@
from recommendation.tasks.task_recommend import recommend # noqa

Просмотреть файл

@ -1,6 +1,6 @@
from app.factory import create_queue
from app.memcached import memcached
from app.search.recommendation import SearchRecommendation
from recommendation.factory import create_queue
from recommendation.memcached import memcached
from recommendation.search.recommendation import SearchRecommendation
queue = create_queue()

Просмотреть файл

Просмотреть файл

@ -3,7 +3,7 @@ from unittest import TestCase
from mock import patch
from nose.tools import eq_
from app.tests.memcached import mock_memcached
from recommendation.tests.memcached import mock_memcached
KEY = 'query_key'
@ -13,12 +13,13 @@ RESULTS = {
}
@patch('app.tasks.task_recommend.memcached', mock_memcached)
@patch('recommendation.tasks.task_recommend.memcached', mock_memcached)
class TestRecommendTask(TestCase):
@patch('app.search.recommendation.SearchRecommendation.do_search')
@patch('recommendation.search.recommendation.SearchRecommendation'
'.do_search')
def test_recommend(self, mock_do_search):
from app.tasks.task_recommend import recommend
from recommendation.tasks.task_recommend import recommend
mock_do_search.return_value = RESULTS
results = recommend.apply(args=[QUERY, KEY]).get()
eq_(results, RESULTS)

Просмотреть файл

Просмотреть файл

@ -5,12 +5,12 @@ class FakeMemcached(object):
`memorize` decorator like so:
from unittest.mock import patch
from app.tests.memcached import mock_memcached
from recommendation.tests.memcached import mock_memcached
def tearDown(self):
mock_memcached.flush_all()
@patch('app.memorize.memcached', mock_memcached)
@patch('recommendation.memorize.memcached', mock_memcached)
def test_foo(self):
assert(True)
"""

Просмотреть файл

@ -0,0 +1,19 @@
from unittest import TestCase
from celery.app.base import Celery
from nose.tools import eq_, ok_
from recommendation.celery import celery
class TestCeleryApp(TestCase):
def test_celery_app(self):
from recommendation import celery as celery_module
ok_(hasattr(celery_module, 'celery'))
def test_celery_app_type(self):
eq_(type(celery), Celery)
def test_celery_app_singleton(self):
from recommendation.celery import celery as celery_2
eq_(id(celery), id(celery_2))

Просмотреть файл

@ -1,7 +1,7 @@
from nose.tools import ok_
from app.cors import cors_headers
from app.tests.util import AppTestCase
from recommendation.cors import cors_headers
from recommendation.tests.util import AppTestCase
class TestCORS(AppTestCase):

Просмотреть файл

@ -4,16 +4,16 @@ from celery.app.base import Celery
from mock import patch
from nose.tools import eq_, ok_
from app.cors import cors_headers
from app.factory import create_app, create_queue
from app.tests.util import AppTestCase
from recommendation.cors import cors_headers
from recommendation.factory import create_app, create_queue
from recommendation.tests.util import AppTestCase
BROKER_URL = 'http://celery.carrots/broccoli'
class TestCreateApp(TestCase):
@patch('app.factory.conf.CELERY_BROKER_URL', BROKER_URL)
@patch('recommendation.factory.conf.CELERY_BROKER_URL', BROKER_URL)
def test_create_app(self):
app = create_app()
ok_(cors_headers in app.after_request_funcs[None])

Просмотреть файл

@ -4,8 +4,8 @@ from flask.ext.testing import TestCase as FlaskTestCase
from mock import patch
from nose.tools import eq_, ok_
from app.cors import cors_headers
from app.tests.memcached import mock_memcached
from recommendation.cors import cors_headers
from recommendation.tests.memcached import mock_memcached
KEY = 'query_key'
@ -17,7 +17,7 @@ EXCEPTION_ARGS = ['args']
EXCEPTION = RuntimeError(*EXCEPTION_ARGS)
@patch('app.tasks.task_recommend.memcached', mock_memcached)
@patch('recommendation.tasks.task_recommend.memcached', mock_memcached)
class TestApp(FlaskTestCase):
debug = False
@ -25,7 +25,7 @@ class TestApp(FlaskTestCase):
mock_memcached.flush_all()
def create_app(self):
from app.main import application
from recommendation.main import application
app = application
app.config['DEBUG'] = self.debug
app.config['PRESERVE_CONTEXT_ON_EXCEPTION'] = False
@ -55,21 +55,21 @@ class TestApp(FlaskTestCase):
def test_no_query(self):
eq_(self._get('/').status_code, 400)
@patch('app.tasks.task_recommend.memcached.get')
@patch('recommendation.tasks.task_recommend.memcached.get')
def test_exception(self, mock_get):
mock_get.side_effect = EXCEPTION
response = self._query(QUERY)
eq_(response.status_code, 500)
eq_(response.json, {})
@patch('app.tasks.task_recommend.memcached.get')
@patch('recommendation.tasks.task_recommend.memcached.get')
def test_cache_hit(self, mock_get):
mock_get.return_value = RESULTS
eq_(self._query(QUERY).status_code, 200)
eq_(self._query(QUERY).json, RESULTS)
@patch('app.tasks.task_recommend.memcached.get')
@patch('app.tasks.task_recommend.recommend.delay')
@patch('recommendation.tasks.task_recommend.memcached.get')
@patch('recommendation.tasks.task_recommend.recommend.delay')
def test_cache_miss(self, mock_delay, mock_get):
mock_get.return_value = None
eq_(self._query(QUERY).status_code, 202)
@ -79,8 +79,8 @@ class TestApp(FlaskTestCase):
class TestAppDebug(TestApp):
debug = True
@patch('app.tasks.task_recommend.memcached.get')
@patch('app.tasks.task_recommend.recommend.delay')
@patch('recommendation.tasks.task_recommend.memcached.get')
@patch('recommendation.tasks.task_recommend.recommend.delay')
def test_exception(self, mock_delay, mock_get):
mock_get.side_effect = EXCEPTION
response = self._query(QUERY)

Просмотреть файл

@ -5,8 +5,8 @@ from mock import patch
from nose.tools import eq_, ok_
from wrapt import ObjectProxy
from app.memorize import memorize, MemorizedObject
from app.tests.memcached import mock_memcached
from recommendation.memorize import memorize, MemorizedObject
from recommendation.tests.memcached import mock_memcached
PREFIX = 'my_prefix'
@ -58,7 +58,7 @@ class TestMemorizedObject(TestCase):
self.instance.cache_key = True
@patch('app.memorize.memcached', mock_memcached)
@patch('recommendation.memorize.memcached', mock_memcached)
class TestMemorize(TestCase):
def setUp(self):
self.obj = SampleObject()

Просмотреть файл

@ -0,0 +1,24 @@
from unittest import TestCase
from flask.app import Flask
from nose.tools import eq_, ok_
class TestCeleryApp(TestCase):
def test_wsgi_app(self):
from recommendation import wsgi as wsgi_module
ok_(hasattr(wsgi_module, 'application'))
ok_(hasattr(wsgi_module, 'celery'))
def test_wsgi_app_type(self):
from recommendation.wsgi import application
eq_(type(application), Flask)
def test_wsgi_app_singleton(self):
"""
Tests that multiple imports of the application singleton have the same
memory ID (i.e. is the same object).
"""
from recommendation.wsgi import application as app1
from recommendation.wsgi import application as app2
eq_(id(app1), id(app2))

Просмотреть файл

@ -1,6 +1,6 @@
from flask.ext.testing import TestCase as FlaskTestCase
from app.factory import create_app
from recommendation.factory import create_app
class AppTestCase(FlaskTestCase):

Просмотреть файл

Просмотреть файл

@ -2,8 +2,8 @@ import hashlib
from flask import abort, current_app, Blueprint, jsonify, request
from app import conf
from app.memcached import memcached
from recommendation import conf
from recommendation.memcached import memcached
main = Blueprint('main', __name__)
@ -11,7 +11,7 @@ main = Blueprint('main', __name__)
@main.route('/')
def view():
from app.tasks.task_recommend import recommend
from recommendation.tasks.task_recommend import recommend
query = request.args.get('q')
if not query:
abort(400)

Просмотреть файл

@ -0,0 +1,11 @@
from os import path, pardir
from flask import Blueprint, send_file
static = Blueprint('static', __name__)
STATIC_DIR = path.join(path.dirname(path.abspath(__file__)), pardir, 'static')
@static.route('/contribute.json')
def lbheartbeat():
return send_file(path.join(STATIC_DIR, 'contribute.json'))

Просмотреть файл

@ -2,7 +2,7 @@ from celery.app.control import Control
from flask import abort, Blueprint
from redis.exceptions import ConnectionError as RedisConnectionError
from app.memcached import memcached
from recommendation.memcached import memcached
status = Blueprint('status', __name__)
@ -32,7 +32,7 @@ def redis_status():
Since our application should not have access to the Redis server, we test
this by instantiating a Celery Control and attempting to ping it.
"""
from app.factory import create_queue
from recommendation.factory import create_queue
try:
Control(app=create_queue()).ping(timeout=1)
except RedisConnectionError:
@ -44,7 +44,7 @@ def celery_status():
Raises ServiceDown if any Celery worker servers are down, if any clusters
have no workers, or if any workers are down.
"""
from app.factory import create_queue
from recommendation.factory import create_queue
clusters = Control(app=create_queue()).ping(timeout=1)
if not clusters:
raise ServiceDown()

Просмотреть файл

Просмотреть файл

@ -3,9 +3,9 @@ from urllib.parse import urlencode
from mock import patch
from nose.tools import eq_, ok_
from app.cors import cors_headers
from app.tests.memcached import mock_memcached
from app.tests.util import AppTestCase
from recommendation.cors import cors_headers
from recommendation.tests.memcached import mock_memcached
from recommendation.tests.util import AppTestCase
KEY = 'query_key'
@ -17,7 +17,7 @@ EXCEPTION_ARGS = ['args']
EXCEPTION = RuntimeError(*EXCEPTION_ARGS)
@patch('app.tasks.task_recommend.memcached', mock_memcached)
@patch('recommendation.tasks.task_recommend.memcached', mock_memcached)
class TestMain(AppTestCase):
debug = False
@ -48,21 +48,21 @@ class TestMain(AppTestCase):
def test_no_query(self):
eq_(self._get('/').status_code, 400)
@patch('app.tasks.task_recommend.memcached.get')
@patch('recommendation.tasks.task_recommend.memcached.get')
def test_exception(self, mock_get):
mock_get.side_effect = EXCEPTION
response = self._query(QUERY)
eq_(response.status_code, 500)
eq_(response.json, {})
@patch('app.tasks.task_recommend.memcached.get')
@patch('recommendation.tasks.task_recommend.memcached.get')
def test_cache_hit(self, mock_get):
mock_get.return_value = RESULTS
eq_(self._query(QUERY).status_code, 200)
eq_(self._query(QUERY).json, RESULTS)
@patch('app.tasks.task_recommend.memcached.get')
@patch('app.tasks.task_recommend.recommend.delay')
@patch('recommendation.tasks.task_recommend.memcached.get')
@patch('recommendation.tasks.task_recommend.recommend.delay')
def test_cache_miss(self, mock_delay, mock_get):
mock_get.return_value = None
eq_(self._query(QUERY).status_code, 202)
@ -72,8 +72,8 @@ class TestMain(AppTestCase):
class TestMainDebug(TestMain):
debug = True
@patch('app.tasks.task_recommend.memcached.get')
@patch('app.tasks.task_recommend.recommend.delay')
@patch('recommendation.tasks.task_recommend.memcached.get')
@patch('recommendation.tasks.task_recommend.recommend.delay')
def test_exception(self, mock_delay, mock_get):
mock_get.side_effect = EXCEPTION
response = self._query(QUERY)

Просмотреть файл

@ -0,0 +1,15 @@
import json
from os import path
from nose.tools import eq_
from recommendation.tests.util import AppTestCase
from recommendation.views.static import STATIC_DIR
class TestStaticViews(AppTestCase):
def test_contribute(self):
response = self.client.get('/contribute.json')
eq_(response.status_code, 200)
with open(path.join(STATIC_DIR, 'contribute.json')) as file_data:
eq_(json.load(file_data), response.json)

Просмотреть файл

@ -2,9 +2,10 @@ from mock import patch
from nose.tools import eq_, ok_
from redis.exceptions import ConnectionError as RedisError
from app.views.status import (celery_status, memcached_status, redis_status,
ServiceDown)
from app.tests.util import AppTestCase
from recommendation.views.status import (celery_status, memcached_status,
redis_status, ServiceDown)
from recommendation.tests.util import AppTestCase
MEMCACHED_WORKER_BAD = {'not ok': 'not pong'}
MEMCACHED_WORKER_OK = {'ok': 'pong'}
@ -32,9 +33,9 @@ class TestStatusViews(AppTestCase):
eq_(response.status_code, 200)
ok_(not response.data)
@patch('app.views.status.celery_status')
@patch('app.views.status.memcached_status')
@patch('app.views.status.redis_status')
@patch('recommendation.views.status.celery_status')
@patch('recommendation.views.status.memcached_status')
@patch('recommendation.views.status.redis_status')
def test_heartbeat_pass(self, mock_celery, mock_memcached, mock_redis):
response = self.client.get('/__heartbeat__')
eq_(response.status_code, 200)
@ -42,18 +43,18 @@ class TestStatusViews(AppTestCase):
eq_(mock_memcached.call_count, 1)
eq_(mock_redis.call_count, 1)
@patch('app.views.status.celery_status')
@patch('app.views.status.memcached_status')
@patch('app.views.status.redis_status')
@patch('recommendation.views.status.celery_status')
@patch('recommendation.views.status.memcached_status')
@patch('recommendation.views.status.redis_status')
def test_heartbeat_celery(self, mock_celery, mock_memcached, mock_redis):
mock_celery.side_effect = ServiceDown
response = self.client.get('/__heartbeat__')
eq_(response.status_code, 500)
eq_(mock_celery.call_count, 1)
@patch('app.views.status.celery_status')
@patch('app.views.status.memcached_status')
@patch('app.views.status.redis_status')
@patch('recommendation.views.status.celery_status')
@patch('recommendation.views.status.memcached_status')
@patch('recommendation.views.status.redis_status')
def test_heartbeat_memcached(self, mock_celery, mock_memcached,
mock_redis):
mock_memcached.side_effect = ServiceDown
@ -61,57 +62,57 @@ class TestStatusViews(AppTestCase):
eq_(response.status_code, 500)
eq_(mock_memcached.call_count, 1)
@patch('app.views.status.celery_status')
@patch('app.views.status.memcached_status')
@patch('app.views.status.redis_status')
@patch('recommendation.views.status.celery_status')
@patch('recommendation.views.status.memcached_status')
@patch('recommendation.views.status.redis_status')
def test_heartbeat_redis(self, mock_celery, mock_memcached, mock_redis):
mock_redis.side_effect = ServiceDown
response = self.client.get('/__heartbeat__')
eq_(response.status_code, 500)
eq_(mock_redis.call_count, 1)
@patch('app.views.status.memcached.set')
@patch('recommendation.views.status.memcached.set')
def test_memcached_status_pass(self, mock_set):
mock_set.return_value = True
memcached_status()
self.assert_(True)
@patch('app.views.status.memcached.set')
@patch('recommendation.views.status.memcached.set')
def test_memcached_status_fail(self, mock_set):
mock_set.return_value = 0
with self.assertRaises(ServiceDown):
memcached_status()
@patch('app.views.status.Control.ping')
@patch('recommendation.views.status.Control.ping')
def test_redis_status_pass(self, mock_ping):
redis_status()
self.assert_(True)
@patch('app.views.status.Control.ping')
@patch('recommendation.views.status.Control.ping')
def test_redis_status_fail(self, mock_ping):
mock_ping.side_effect = RedisError
with self.assertRaises(ServiceDown):
redis_status()
@patch('app.views.status.Control.ping')
@patch('recommendation.views.status.Control.ping')
def test_celery_status_pass(self, mock_ping):
mock_ping.return_value = MEMCACHED_PING_OK
celery_status()
self.assert_(True)
@patch('app.views.status.Control.ping')
@patch('recommendation.views.status.Control.ping')
def test_celery_status_no_workers(self, mock_ping):
mock_ping.return_value = MEMCACHED_PING_NO_WORKERS
with self.assertRaises(ServiceDown):
celery_status()
@patch('app.views.status.Control.ping')
@patch('recommendation.views.status.Control.ping')
def test_celery_status_no_clusters(self, mock_ping):
mock_ping.return_value = MEMCACHED_PING_NO_CLUSTERS
with self.assertRaises(ServiceDown):
celery_status()
@patch('app.views.status.Control.ping')
@patch('recommendation.views.status.Control.ping')
def test_celery_status_workers_down(self, mock_ping):
mock_ping.return_value = MEMCACHED_PING_BAD
with self.assertRaises(ServiceDown):

4
recommendation/wsgi.py Normal file
Просмотреть файл

@ -0,0 +1,4 @@
from recommendation.celery import celery # noqa
from recommendation.factory import create_app
application = create_app()

Просмотреть файл

@ -12,6 +12,7 @@ Jinja2==2.8
MarkupSafe==0.23
mock==1.3.0
nose==1.3.7
nose-exclude==0.4.1
oauth2==1.9.0.post1
redis==2.10.5
requests==2.9.1

Просмотреть файл

@ -1,12 +0,0 @@
import os
import sys
from app import conf
from app.main import app
sys.path.append(os.path.dirname(__file__))
if __name__ == '__main__':
print('Running server on http://%s:%d' % (conf.HOST, conf.PORT))
app.run(debug=conf.DEBUG, host=conf.HOST, port=conf.PORT)

Просмотреть файл

@ -1,2 +1,2 @@
[aliases]
test=nosetests --with-coverage --cover-package=app --cover-min-percentage=100 --exclude=app/src
test=nosetests --with-coverage --cover-package=recommendation --cover-min-percentage=100 --exclude=recommendation/src