зеркало из https://github.com/mozilla/esFrontLine.git
fix the slow server test
This commit is contained in:
Kyle Lahnakoski
Родитель
65012249a7
Коммит
52474a4372
|
|
@ -90,7 +90,7 @@ def catch_all(path):
|
|||
timeout=5
|
||||
)
|
||||
except Exception as e:
|
||||
pass
|
||||
Log.warning("can not forward request", cause=e)
|
||||
|
||||
if DEBUG:
|
||||
Log.note(
|
||||
|
|
|
|||
|
|
@ -79,8 +79,6 @@ class HawkAuth(object):
|
|||
except Exception as e:
|
||||
Log.error(AUTH_EXCEPTION, reason="unexpected", cause=e)
|
||||
|
||||
|
||||
|
||||
def check_resource(self, user_id, resource):
|
||||
'''
|
||||
Check the resource is allowed by comparing with resources for the user
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
{
|
||||
"elasticsearch":[{
|
||||
"elasticsearch":{
|
||||
"host":"http://localhost",
|
||||
"port":9299
|
||||
}],
|
||||
},
|
||||
"flask":{
|
||||
"host":"0.0.0.0",
|
||||
"port":9298,
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
#
|
||||
from __future__ import division
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from flask import Flask, Response, abort
|
||||
from mo_logs import startup, constants, Log
|
||||
|
||||
from mo_logs.strings import unicode2utf8, expand_template
|
||||
from mo_threads import Till
|
||||
|
||||
SLOW_PORT = 9299
|
||||
RATE = 4.0 # per second
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
|
||||
@app.route('/', defaults={'path': ''}, methods=['GET'])
|
||||
@app.route('/<path:path>', methods=['GET'])
|
||||
def serve_slowly(path):
|
||||
def octoberfest():
|
||||
for bb in range(99, 2, -1):
|
||||
Log.note("emit {{i}}", i=bb)
|
||||
yield (b"0" * 65535) + b"\n" # ENOUGH TO FILL THE INCOMING BUFFER
|
||||
Till(seconds=1.0 / RATE).wait()
|
||||
yield unicode2utf8(expand_template("{{num}} bottles of beer on the wall! {{num}} bottles of beer! Take one down, pass it around! {{less}} bottles of beer on he wall!\n", {
|
||||
"num": bb,
|
||||
"less": bb - 1
|
||||
}))
|
||||
yield (b"0" * 65535) + b"\n" # ENOUGH TO FILL THE INCOMING BUFFER
|
||||
yield unicode2utf8(u"2 bottles of beer on the wall! 2 bottles of beer! Take one down, pass it around! 1 bottle of beer on he wall!\n")
|
||||
yield (b"0" * 65535) + b"\n" # ENOUGH TO FILL THE INCOMING BUFFER
|
||||
yield unicode2utf8(u"1 bottle of beer on the wall! 1 bottle of beer! Take one down, pass it around! 0 bottles of beer on he wall.\n")
|
||||
|
||||
try:
|
||||
# FORWARD RESPONSE
|
||||
return Response(
|
||||
octoberfest(),
|
||||
direct_passthrough=True, # FOR STREAMING
|
||||
status=200
|
||||
)
|
||||
except Exception as e:
|
||||
abort(400)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
settings = startup.read_settings(filename="tests/config/slow_server.json")
|
||||
|
||||
app.run(
|
||||
host="0.0.0.0",
|
||||
port=settings.elasticsearch.port,
|
||||
debug=False,
|
||||
threaded=False,
|
||||
processes=1
|
||||
)
|
||||
app.run(**settings.flask)
|
||||
|
|
@ -7,68 +7,70 @@
|
|||
#
|
||||
# Author: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
import time
|
||||
import unittest
|
||||
|
||||
import requests
|
||||
from flask import Flask, Response
|
||||
from werkzeug.exceptions import abort
|
||||
|
||||
from esFrontLine.app import stream
|
||||
from mo_logs import Log
|
||||
from mo_logs.strings import expand_template, unicode2utf8
|
||||
from mo_threads import Signal, Thread, Till, Process
|
||||
from mo_logs import Log, startup
|
||||
from mo_threads import Signal, Thread, Process
|
||||
from mo_times import Timer
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
WHITELISTED = "public_bugs" # ENSURE THIS IS IN THE slow_server_settings.json WHITELIST
|
||||
RUN_SLOW_SERVER = True
|
||||
RUN_PROXY = True
|
||||
|
||||
WHITELISTED = "public_bugs" # YOU MUST ENSURE THIS IS IN THE slow_server.json WHITELIST
|
||||
PATH = '/'+WHITELISTED+'/_mapping'
|
||||
SLOW_PORT = 9299
|
||||
PROXY_PORT = 9298
|
||||
RATE = 4.0 # per second
|
||||
proxy_is_ready = Signal()
|
||||
|
||||
server_is_ready = Signal()
|
||||
proxy_is_ready = Signal()
|
||||
settings = startup.read_settings(filename="tests/config/slow_server.json")
|
||||
|
||||
|
||||
@app.route('/', defaults={'path': ''}, methods=['GET'])
|
||||
@app.route('/<path:path>', methods=['GET'])
|
||||
def serve_slowly(path):
|
||||
def octoberfest():
|
||||
for bb in range(99, 2, -1):
|
||||
yield ("0"*65535)+"\n" # ENOUGH TO FILL THE INCOMING BUFFER
|
||||
Till(seconds=1.0/RATE).wait()
|
||||
yield unicode2utf8(expand_template("{{num}} bottles of beer on the wall! {{num}} bottles of beer! Take one down, pass it around! {{less}} bottles of beer on he wall!\n", {
|
||||
"num": bb,
|
||||
"less": bb - 1
|
||||
}))
|
||||
yield ("0"*65535)+"\n" # ENOUGH TO FILL THE INCOMING BUFFER
|
||||
yield unicode2utf8(u"2 bottles of beer on the wall! 2 bottles of beer! Take one down, pass it around! 1 bottle of beer on he wall!\n")
|
||||
yield ("0"*65535)+"\n" # ENOUGH TO FILL THE INCOMING BUFFER
|
||||
yield unicode2utf8(u"1 bottle of beer on the wall! 1 bottle of beer! Take one down, pass it around! 0 bottles of beer on he wall.\n")
|
||||
class TestSlowSever(unittest.TestCase):
|
||||
|
||||
try:
|
||||
# FORWARD RESPONSE
|
||||
return Response(
|
||||
octoberfest(),
|
||||
direct_passthrough=True, #FOR STREAMING
|
||||
status=200
|
||||
)
|
||||
except Exception as e:
|
||||
abort(400)
|
||||
def test_slow_streaming(self):
|
||||
"""
|
||||
TEST THAT THE app ACTUALLY STREAMS. WE SHOULD GET A RESPONSE BEFORE THE SERVER
|
||||
FINISHES DELIVERING
|
||||
"""
|
||||
slow_server_thread = Thread.run("run slow server", _run_slow_server)
|
||||
proxy_thread = Thread.run("run proxy", _run_esFrontline)
|
||||
|
||||
try:
|
||||
proxy_is_ready.wait()
|
||||
server_is_ready.wait()
|
||||
|
||||
with Timer("measure response times") as timer:
|
||||
response = requests.get("http://localhost:"+str(settings.flask.port)+PATH, stream=True)
|
||||
for i, data in enumerate(stream(response.raw)):
|
||||
Log.note("CLIENT GOT RESPONSE:\n{{data|indent}}", {"data": data})
|
||||
duration = timer.duration.seconds
|
||||
if i == 0:
|
||||
self.assertLess(duration, 10, "should have something by now")
|
||||
# self.assertEqual(response.status_code, 200, "Expecting a positive response")
|
||||
self.assertGreater(timer.duration.seconds, 10, "expecting slkow server to talk a while")
|
||||
finally:
|
||||
slow_server_thread.please_stop.go()
|
||||
proxy_thread.please_stop.go()
|
||||
|
||||
|
||||
def run_slow_server(please_stop):
|
||||
def _run_slow_server(please_stop):
|
||||
if not RUN_SLOW_SERVER:
|
||||
server_is_ready.go()
|
||||
return
|
||||
|
||||
proc = Process(
|
||||
"slow server",
|
||||
["python", "tests\\test_slow_server.py"],
|
||||
["python", "tests/slow_server.py"],
|
||||
debug=True
|
||||
)
|
||||
|
||||
while not please_stop:
|
||||
line = proc.stdout.readline()
|
||||
line = proc.stderr.pop().decode('utf8')
|
||||
if not line:
|
||||
continue
|
||||
if line.find(" * Running on") >= 0:
|
||||
if " * Running on" in line:
|
||||
server_is_ready.go()
|
||||
Log.note("SLOW SERVER: "+line)
|
||||
|
||||
|
|
@ -76,60 +78,25 @@ def run_slow_server(please_stop):
|
|||
proc.join()
|
||||
|
||||
|
||||
def run_proxy(please_stop):
|
||||
def _run_esFrontline(please_stop):
|
||||
if not RUN_PROXY:
|
||||
proxy_is_ready.go()
|
||||
return
|
||||
|
||||
proc = Process(
|
||||
"slow server",
|
||||
["python", "esFrontLine\\app.py", "--settings", "tests/resources/slow_server_settings.json"],
|
||||
["python", "esFrontLine/app.py", "--config", "tests/config/slow_server.json"],
|
||||
debug=True
|
||||
)
|
||||
|
||||
while not please_stop:
|
||||
line = proc.stdout.readline()
|
||||
line = proc.stderr.pop().decode('utf8')
|
||||
if not line:
|
||||
continue
|
||||
if line.find(" * Running on") >= 0:
|
||||
if " * Running on" in line:
|
||||
proxy_is_ready.go()
|
||||
Log.note("PROXY: {{line}}", {"line": line.strip()})
|
||||
|
||||
proc.stop()
|
||||
proc.join()
|
||||
|
||||
|
||||
def test_slow_streaming():
|
||||
"""
|
||||
TEST THAT THE app ACTUALLY STREAMS. WE SHOULD GET A RESPONSE BEFORE THE SERVER
|
||||
FINISHES DELIVERING
|
||||
"""
|
||||
slow_server_thread = Thread.run("run slow server", run_slow_server)
|
||||
proxy_thread = Thread.run("run proxy", run_proxy)
|
||||
|
||||
try:
|
||||
proxy_is_ready.wait()()
|
||||
server_is_ready.wait()()
|
||||
|
||||
start = time.clock()
|
||||
response = requests.get("http://localhost:"+str(PROXY_PORT)+PATH, stream=True)
|
||||
for i, data in enumerate(stream(response.raw)):
|
||||
Log.note("CLIENT GOT RESPONSE:\n{{data|indent}}", {"data": data})
|
||||
end = time.clock()
|
||||
if i == 0 and end - start > 10: # IF WE GET DATA BEFORE 10sec, THEN WE KNOW WE ARE STREAMING
|
||||
Log.error("should have something by now")
|
||||
if response.status_code != 200:
|
||||
Log.error("Expecting a positive response")
|
||||
|
||||
except Exception as e:
|
||||
Log.error("Not expected", e)
|
||||
finally:
|
||||
slow_server_thread.please_stop.go()
|
||||
proxy_thread.please_stop.go()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
#THIS WILL RUN THE SLOW SERVER
|
||||
app.run(
|
||||
host="0.0.0.0",
|
||||
port=SLOW_PORT,
|
||||
debug=False,
|
||||
threaded=False,
|
||||
processes=1
|
||||
)
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче