зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1631806 - [mozproxy] Add a command line interface r=tarek,Bebe
Differential Revision: https://phabricator.services.mozilla.com/D76017
This commit is contained in:
Jim Porter
Родитель
5542e44de3
Коммит
5c759e3542
|
|
@ -1,17 +1,61 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
import json
|
||||
import os
|
||||
import mozinfo
|
||||
import re
|
||||
import signal
|
||||
import threading
|
||||
|
||||
from mozproxy import get_playback
|
||||
from mozproxy.utils import LOG
|
||||
from mozlog import get_proxy_logger
|
||||
from mozperftest.layers import Layer
|
||||
from mozperftest.utils import install_package
|
||||
from mozprocess import ProcessHandler
|
||||
|
||||
|
||||
LOG = get_proxy_logger(component="proxy")
|
||||
HERE = os.path.dirname(__file__)
|
||||
|
||||
|
||||
class OutputHandler(object):
|
||||
def __init__(self):
|
||||
self.proc = None
|
||||
self.port = None
|
||||
self.port_event = threading.Event()
|
||||
|
||||
def __call__(self, line):
|
||||
if not line.strip():
|
||||
return
|
||||
line = line.decode("utf-8", errors="replace")
|
||||
|
||||
try:
|
||||
data = json.loads(line)
|
||||
except ValueError:
|
||||
self.process_output(line)
|
||||
return
|
||||
|
||||
if isinstance(data, dict) and "action" in data:
|
||||
# Retrieve the port number for the proxy server from the logs of
|
||||
# our subprocess.
|
||||
m = re.match(r"Proxy running on port (\d+)", data.get("message", ""))
|
||||
if m:
|
||||
self.port = m.group(1)
|
||||
self.port_event.set()
|
||||
LOG.log_raw(data)
|
||||
else:
|
||||
self.process_output(json.dumps(data))
|
||||
|
||||
def finished(self):
|
||||
self.port_event.set()
|
||||
|
||||
def process_output(self, line):
|
||||
LOG.process_output(self.proc.pid, line)
|
||||
|
||||
def wait_for_port(self):
|
||||
self.port_event.wait()
|
||||
return self.port
|
||||
|
||||
|
||||
class ProxyRunner(Layer):
|
||||
"""Use a proxy
|
||||
"""
|
||||
|
|
@ -22,44 +66,57 @@ class ProxyRunner(Layer):
|
|||
def __init__(self, env, mach_cmd):
|
||||
super(ProxyRunner, self).__init__(env, mach_cmd)
|
||||
self.proxy = None
|
||||
LOG.info = self.info
|
||||
LOG.error = self.error
|
||||
|
||||
def setup(self):
|
||||
pass
|
||||
# Install mozproxy and its vendored deps.
|
||||
mozbase = os.path.join(self.mach_cmd.topsrcdir, "testing", "mozbase")
|
||||
mozproxy_deps = ["mozinfo", "mozlog", "mozproxy"]
|
||||
for i in mozproxy_deps:
|
||||
install_package(self.mach_cmd.virtualenv_manager, os.path.join(mozbase, i))
|
||||
|
||||
def run(self, metadata):
|
||||
self.metadata = metadata
|
||||
|
||||
self.info("Setting up the proxy")
|
||||
# replace with artifacts
|
||||
config = {
|
||||
"run_local": True,
|
||||
"playback_tool": "mitmproxy",
|
||||
"host": "localhost",
|
||||
"binary": self.mach_cmd.get_binary_path(),
|
||||
"obj_path": self.mach_cmd.topobjdir,
|
||||
"platform": mozinfo.os,
|
||||
"playback_files": [os.path.join(HERE, "example.dump")],
|
||||
"app": "firefox",
|
||||
}
|
||||
self.info("setting up the proxy")
|
||||
self.proxy = get_playback(config)
|
||||
if self.proxy is not None:
|
||||
self.proxy.start()
|
||||
port = str(self.proxy.port)
|
||||
prefs = {}
|
||||
prefs["network.proxy.type"] = 1
|
||||
prefs["network.proxy.http"] = "localhost"
|
||||
prefs["network.proxy.http_port"] = port
|
||||
prefs["network.proxy.ssl"] = "localhost"
|
||||
prefs["network.proxy.ssl_port"] = port
|
||||
prefs["network.proxy.no_proxies_on"] = "localhost"
|
||||
browser_prefs = metadata.get_options("browser_prefs")
|
||||
browser_prefs.update(prefs)
|
||||
self.output_handler = OutputHandler()
|
||||
self.proxy = ProcessHandler(
|
||||
[
|
||||
"mozproxy",
|
||||
"--local",
|
||||
"--binary=" + self.mach_cmd.get_binary_path(),
|
||||
"--topsrcdir=" + self.mach_cmd.topsrcdir,
|
||||
"--objdir=" + self.mach_cmd.topobjdir,
|
||||
os.path.join(HERE, "example.dump"),
|
||||
],
|
||||
processOutputLine=self.output_handler,
|
||||
onFinish=self.output_handler.finished,
|
||||
)
|
||||
self.output_handler.proc = self.proxy
|
||||
self.proxy.run()
|
||||
|
||||
# Wait until we've retrieved the proxy server's port number so we can
|
||||
# configure the browser properly.
|
||||
port = self.output_handler.wait_for_port()
|
||||
if port is None:
|
||||
raise ValueError("Unable to retrieve the port number from mozproxy")
|
||||
self.info("Received port number %s from mozproxy" % port)
|
||||
|
||||
prefs = {
|
||||
"network.proxy.type": 1,
|
||||
"network.proxy.http": "localhost",
|
||||
"network.proxy.http_port": port,
|
||||
"network.proxy.ssl": "localhost",
|
||||
"network.proxy.ssl_port": port,
|
||||
"network.proxy.no_proxies_on": "localhost",
|
||||
}
|
||||
browser_prefs = metadata.get_options("browser_prefs")
|
||||
browser_prefs.update(prefs)
|
||||
return metadata
|
||||
|
||||
def teardown(self):
|
||||
if self.proxy is not None:
|
||||
self.proxy.stop()
|
||||
kill_signal = getattr(signal, "CTRL_BREAK_EVENT", signal.SIGINT)
|
||||
os.kill(self.proxy.pid, kill_signal)
|
||||
self.proxy.wait()
|
||||
self.proxy = None
|
||||
|
|
|
|||
|
|
@ -1,12 +1,29 @@
|
|||
#!/usr/bin/env python
|
||||
import mozunit
|
||||
import os
|
||||
import pytest
|
||||
|
||||
from mozbuild.base import MozbuildObject
|
||||
from mozperftest.tests.support import get_running_env
|
||||
from mozperftest.environment import SYSTEM
|
||||
from mozperftest.utils import silence
|
||||
from mozperftest.utils import install_package, silence
|
||||
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
|
||||
|
||||
def test_proxy():
|
||||
@pytest.fixture(scope="module")
|
||||
def install_mozproxy():
|
||||
build = MozbuildObject.from_environment(cwd=here)
|
||||
build.virtualenv_manager.activate()
|
||||
|
||||
mozbase = os.path.join(build.topsrcdir, "testing", "mozbase")
|
||||
mozproxy_deps = ["mozinfo", "mozlog", "mozproxy"]
|
||||
for i in mozproxy_deps:
|
||||
install_package(build.virtualenv_manager, os.path.join(mozbase, i))
|
||||
return build
|
||||
|
||||
|
||||
def test_proxy(install_mozproxy):
|
||||
mach_cmd, metadata, env = get_running_env(proxy=True)
|
||||
system = env.layers[SYSTEM]
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,79 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
from __future__ import absolute_import
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
|
||||
import mozinfo
|
||||
import mozlog.commandline
|
||||
|
||||
from . import get_playback
|
||||
from .utils import LOG, TOOLTOOL_PATHS
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--local", action="store_true",
|
||||
help="run this locally (i.e. not in production)")
|
||||
parser.add_argument("--tool", default="mitmproxy",
|
||||
help="the playback tool to use (default: %(default)s)")
|
||||
parser.add_argument("--host", default="localhost",
|
||||
help="the host to use for the proxy server")
|
||||
parser.add_argument("--binary", required=True,
|
||||
help=("the path to the binary being tested (typically "
|
||||
"firefox)"))
|
||||
parser.add_argument("--topsrcdir", required=True,
|
||||
help="the top of the source directory for this project")
|
||||
parser.add_argument("--objdir", required=True,
|
||||
help="the object directory for this build")
|
||||
parser.add_argument("--app", default="firefox",
|
||||
help="the app being tested (default: %(default)s)")
|
||||
parser.add_argument("playback", nargs="*", help="the playback file to use")
|
||||
|
||||
mozlog.commandline.add_logging_group(parser)
|
||||
|
||||
args = parser.parse_args()
|
||||
mozlog.commandline.setup_logging("mozproxy", args, {"raw": sys.stdout})
|
||||
|
||||
TOOLTOOL_PATHS.append(os.path.join(args.topsrcdir, "python", "mozbuild",
|
||||
"mozbuild", "action", "tooltool.py"))
|
||||
|
||||
if hasattr(signal, "SIGBREAK"):
|
||||
# Terminating on windows is slightly different than other platforms.
|
||||
# On POSIX, we just let Python's default SIGINT handler raise a
|
||||
# KeyboardInterrupt. This doesn't work on Windows, so instead we wait
|
||||
# for a Ctrl+Break event and raise our own KeyboardInterrupt.
|
||||
def handle_sigbreak(sig, frame):
|
||||
raise KeyboardInterrupt()
|
||||
|
||||
signal.signal(signal.SIGBREAK, handle_sigbreak)
|
||||
|
||||
try:
|
||||
playback = get_playback({
|
||||
"run_local": args.local,
|
||||
"playback_tool": args.tool,
|
||||
"host": args.host,
|
||||
"binary": args.binary,
|
||||
"obj_path": args.objdir,
|
||||
"platform": mozinfo.os,
|
||||
"playback_files": args.playback,
|
||||
"app": args.app
|
||||
})
|
||||
playback.start()
|
||||
|
||||
LOG.info("Proxy running on port %d" % playback.port)
|
||||
# Wait for a keyboard interrupt from the caller so we know when to
|
||||
# terminate. We wait using this method to allow Windows to respond to
|
||||
# the Ctrl+Break signal so that we can exit cleanly.
|
||||
while True:
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
LOG.info("Terminating mozproxy")
|
||||
playback.stop()
|
||||
except Exception as e:
|
||||
LOG.error(str(e), exc_info=True)
|
||||
|
|
@ -10,7 +10,7 @@ PACKAGE_NAME = "mozproxy"
|
|||
PACKAGE_VERSION = "1.0"
|
||||
|
||||
# dependencies
|
||||
deps = ["redo"]
|
||||
deps = ["redo", "mozinfo", "mozlog >= 6.0"]
|
||||
|
||||
setup(
|
||||
name=PACKAGE_NAME,
|
||||
|
|
@ -27,8 +27,15 @@ setup(
|
|||
author_email="tools@lists.mozilla.org",
|
||||
url="https://wiki.mozilla.org/Auto-tools/Projects/Mozbase",
|
||||
license="MPL",
|
||||
|
||||
packages=["mozproxy"],
|
||||
install_requires=deps,
|
||||
entry_points={
|
||||
'console_scripts': [
|
||||
'mozproxy=mozproxy.driver:main',
|
||||
],
|
||||
},
|
||||
|
||||
include_package_data=True,
|
||||
zip_safe=False,
|
||||
install_requires=deps,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,3 +2,4 @@
|
|||
subsuite = mozbase
|
||||
[test_proxy.py]
|
||||
[test_utils.py]
|
||||
[test_command_line.py]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,127 @@
|
|||
#!/usr/bin/env python
|
||||
from __future__ import absolute_import
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import threading
|
||||
|
||||
import mozunit
|
||||
import pytest
|
||||
from mozbuild.base import MozbuildObject
|
||||
from mozprocess import ProcessHandler
|
||||
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
|
||||
|
||||
# This is copied from <python/mozperftest/mozperftest/utils.py>. It's copied
|
||||
# instead of imported since mozperfest is Python 3, and this file is
|
||||
# (currently) Python 2.
|
||||
def _install_package(virtualenv_manager, package):
|
||||
from pip._internal.req.constructors import install_req_from_line
|
||||
|
||||
req = install_req_from_line(package)
|
||||
req.check_if_exists(use_user_site=False)
|
||||
# already installed, check if it's in our venv
|
||||
if req.satisfied_by is not None:
|
||||
venv_site_lib = os.path.abspath(
|
||||
os.path.join(virtualenv_manager.bin_path, "..", "lib")
|
||||
)
|
||||
site_packages = os.path.abspath(req.satisfied_by.location)
|
||||
if site_packages.startswith(venv_site_lib):
|
||||
# already installed in this venv, we can skip
|
||||
return
|
||||
virtualenv_manager._run_pip(["install", package])
|
||||
|
||||
|
||||
def _kill_mozproxy(pid):
|
||||
kill_signal = getattr(signal, "CTRL_BREAK_EVENT", signal.SIGINT)
|
||||
os.kill(pid, kill_signal)
|
||||
|
||||
|
||||
class OutputHandler(object):
|
||||
def __init__(self):
|
||||
self.port = None
|
||||
self.port_event = threading.Event()
|
||||
|
||||
def __call__(self, line):
|
||||
if not line.strip():
|
||||
return
|
||||
line = line.decode("utf-8", errors="replace")
|
||||
|
||||
try:
|
||||
data = json.loads(line)
|
||||
except ValueError:
|
||||
return
|
||||
|
||||
if isinstance(data, dict) and "action" in data:
|
||||
# Retrieve the port number for the proxy server from the logs of
|
||||
# our subprocess.
|
||||
m = re.match(r"Proxy running on port (\d+)",
|
||||
data.get("message", ""))
|
||||
if m:
|
||||
self.port = m.group(1)
|
||||
self.port_event.set()
|
||||
|
||||
def finished(self):
|
||||
self.port_event.set()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def install_mozproxy():
|
||||
build = MozbuildObject.from_environment(cwd=here)
|
||||
build.virtualenv_manager.activate()
|
||||
|
||||
mozbase = os.path.join(build.topsrcdir, "testing", "mozbase")
|
||||
mozproxy_deps = ["mozinfo", "mozlog", "mozproxy"]
|
||||
for i in mozproxy_deps:
|
||||
_install_package(build.virtualenv_manager, os.path.join(mozbase, i))
|
||||
return build
|
||||
|
||||
|
||||
def test_help(install_mozproxy):
|
||||
p = ProcessHandler(["mozproxy", "--help"])
|
||||
p.run()
|
||||
assert p.wait() == 0
|
||||
|
||||
|
||||
def test_run(install_mozproxy):
|
||||
build = install_mozproxy
|
||||
output_handler = OutputHandler()
|
||||
p = ProcessHandler(
|
||||
["mozproxy",
|
||||
"--local",
|
||||
"--binary=firefox",
|
||||
"--topsrcdir=" + build.topsrcdir,
|
||||
"--objdir=" + build.topobjdir],
|
||||
processOutputLine=output_handler,
|
||||
onFinish=output_handler.finished,
|
||||
)
|
||||
p.run()
|
||||
# The first time we run mozproxy, we need to fetch mitmproxy, which can
|
||||
# take a while...
|
||||
assert output_handler.port_event.wait(120) is True
|
||||
_kill_mozproxy(p.pid)
|
||||
|
||||
assert p.wait(10) == 0
|
||||
assert output_handler.port is not None
|
||||
|
||||
|
||||
def test_failure(install_mozproxy):
|
||||
output_handler = OutputHandler()
|
||||
p = ProcessHandler(
|
||||
["mozproxy",
|
||||
"--local",
|
||||
# Exclude some options here to trigger a command-line error.
|
||||
os.path.join(here, "example.dump")],
|
||||
processOutputLine=output_handler,
|
||||
onFinish=output_handler.finished,
|
||||
)
|
||||
p.run()
|
||||
assert output_handler.port_event.wait(10) is True
|
||||
assert p.wait(10) == 2
|
||||
assert output_handler.port is None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
mozunit.main(runwith="pytest")
|
||||
Загрузка…
Ссылка в новой задаче