зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1614565 - [Mozproxy] Add mitmproxy 5.0.1 capability to mozproxy r=tarek,perftest-reviewers,AlexandruIonescu
Differential Revision: https://phabricator.services.mozilla.com/D62420 --HG-- rename : testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay-4.0.4.py => testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay.py extra : moz-landing-system : lando
This commit is contained in:
Florin Strugariu
Родитель
3baed30511
Коммит
2fd5275d13
|
|
@ -84,7 +84,9 @@ POLICIES_CONTENT_OFF = """{
|
|||
class Mitmproxy(Playback):
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
self.host = "127.0.0.1" if 'localhost' in self.config["host"] else self.config["host"]
|
||||
self.host = (
|
||||
"127.0.0.1" if "localhost" in self.config["host"] else self.config["host"]
|
||||
)
|
||||
self.port = None
|
||||
self.mitmproxy_proc = None
|
||||
self.mitmdump_path = None
|
||||
|
|
@ -95,14 +97,21 @@ class Mitmproxy(Playback):
|
|||
)
|
||||
|
||||
if self.config.get("playback_version") is None:
|
||||
LOG.info("mitmproxy was not provided with a 'playback_version' "
|
||||
"Using default playback version: 4.0.4")
|
||||
LOG.info(
|
||||
"mitmproxy was not provided with a 'playback_version' "
|
||||
"Using default playback version: 4.0.4"
|
||||
)
|
||||
self.config["playback_version"] = "4.0.4"
|
||||
|
||||
if self.config.get("playback_binary_manifest") is None:
|
||||
LOG.info("mitmproxy was not provided with a 'playback_binary_manifest' "
|
||||
"Using default playback_binary_manifest")
|
||||
self.config["playback_binary_manifest"] = "mitmproxy-rel-bin-4.0.4-{platform}.manifest"
|
||||
LOG.info(
|
||||
"mitmproxy was not provided with a 'playback_binary_manifest' "
|
||||
"Using default playback_binary_manifest"
|
||||
)
|
||||
self.config["playback_binary_manifest"] = (
|
||||
"mitmproxy-rel-bin-%s-{platform}.manifest"
|
||||
% self.config["playback_version"]
|
||||
)
|
||||
|
||||
# mozproxy_dir is where we will download all mitmproxy required files
|
||||
# when running locally it comes from obj_path via mozharness/mach
|
||||
|
|
@ -127,6 +136,9 @@ class Mitmproxy(Playback):
|
|||
# where to get the data
|
||||
os.environ["MOZPROXY_DIR"] = self.mozproxy_dir
|
||||
|
||||
LOG.info("Playback tool: %s" % self.config["playback_tool"])
|
||||
LOG.info("Playback tool version: %s" % self.config["playback_version"])
|
||||
|
||||
def start(self):
|
||||
# go ahead and download and setup mitmproxy
|
||||
self.download()
|
||||
|
|
@ -151,9 +163,12 @@ class Mitmproxy(Playback):
|
|||
|
||||
# generate the mitmdump_path
|
||||
self.mitmdump_path = os.path.normpath(
|
||||
os.path.join(self.mozproxy_dir, "mitmdump-%s" %
|
||||
self.config["playback_version"],
|
||||
"mitmdump"))
|
||||
os.path.join(
|
||||
self.mozproxy_dir,
|
||||
"mitmdump-%s" % self.config["playback_version"],
|
||||
"mitmdump",
|
||||
)
|
||||
)
|
||||
|
||||
# Check if mitmproxy bin exists
|
||||
if os.path.exists(self.mitmdump_path):
|
||||
|
|
@ -167,8 +182,8 @@ class Mitmproxy(Playback):
|
|||
|
||||
LOG.info("downloading mitmproxy binary")
|
||||
tooltool_download(
|
||||
transformed_manifest, self.config["run_local"],
|
||||
download_path)
|
||||
transformed_manifest, self.config["run_local"], download_path
|
||||
)
|
||||
|
||||
if "playback_pageset_manifest" in self.config:
|
||||
# we use one pageset for all platforms
|
||||
|
|
@ -213,44 +228,55 @@ class Mitmproxy(Playback):
|
|||
command = [mitmdump_path]
|
||||
|
||||
# add proxy host and port options
|
||||
command.extend(["--listen-host", self.host,
|
||||
"--listen-port", str(self.port)])
|
||||
command.extend(["--listen-host", self.host, "--listen-port", str(self.port)])
|
||||
|
||||
if "playback_tool_args" in self.config:
|
||||
LOG.info("Staring Proxy using provided command line!")
|
||||
command.extend(self.config["playback_tool_args"])
|
||||
elif "playback_files" in self.config:
|
||||
script = os.path.join(
|
||||
os.path.dirname(os.path.realpath(__file__)), "scripts",
|
||||
"alternate-server-replay-{}.py".format(
|
||||
self.config["playback_version"]))
|
||||
os.path.dirname(os.path.realpath(__file__)),
|
||||
"scripts",
|
||||
"alternate-server-replay.py",
|
||||
)
|
||||
recording_paths = [
|
||||
normalize_path(recording_path)
|
||||
for recording_path in self.config["playback_files"]
|
||||
]
|
||||
|
||||
recording_paths = [normalize_path(recording_path)
|
||||
for recording_path in self.config["playback_files"]]
|
||||
|
||||
if self.config["playback_version"] == "4.0.4":
|
||||
if self.config["playback_version"] in ["4.0.4", "5.0.1"]:
|
||||
args = [
|
||||
"-v",
|
||||
"--set", "upstream_cert=false",
|
||||
"--set", "upload_dir=" + normalize_path(self.upload_dir),
|
||||
"--set", "websocket=false",
|
||||
"--set", "server_replay_files={}".format(",".join(recording_paths)),
|
||||
"--scripts", normalize_path(script),
|
||||
"--set",
|
||||
"upstream_cert=false",
|
||||
"--set",
|
||||
"upload_dir=" + normalize_path(self.upload_dir),
|
||||
"--set",
|
||||
"websocket=false",
|
||||
"--set",
|
||||
"server_replay_files={}".format(",".join(recording_paths)),
|
||||
"--scripts",
|
||||
normalize_path(script),
|
||||
]
|
||||
command.extend(args)
|
||||
else:
|
||||
raise Exception("Mitmproxy version is unknown!")
|
||||
|
||||
else:
|
||||
raise Exception("Mitmproxy can't start playback! Playback settings missing.")
|
||||
raise Exception(
|
||||
"Mitmproxy can't start playback! Playback settings missing."
|
||||
)
|
||||
|
||||
LOG.info("Starting mitmproxy playback using env path: %s" % env["PATH"])
|
||||
LOG.info("Starting mitmproxy playback using command: %s" % " ".join(command))
|
||||
# to turn off mitmproxy log output, use these params for Popen:
|
||||
# Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
|
||||
self.mitmproxy_proc = ProcessHandler(
|
||||
command, logfile=os.path.join(self.upload_dir, "mitmproxy.log"),
|
||||
env=env, processStderrLine=LOG.error, storeOutput=False
|
||||
command,
|
||||
logfile=os.path.join(self.upload_dir, "mitmproxy.log"),
|
||||
env=env,
|
||||
processStderrLine=LOG.error,
|
||||
storeOutput=False,
|
||||
)
|
||||
self.mitmproxy_proc.run()
|
||||
end_time = time.time() + MITMDUMP_COMMAND_TIMEOUT
|
||||
|
|
@ -295,9 +321,12 @@ class Mitmproxy(Playback):
|
|||
|
||||
if mozinfo.os == "win":
|
||||
from mozprocess.winprocess import ERROR_CONTROL_C_EXIT # noqa
|
||||
|
||||
if exit_code == ERROR_CONTROL_C_EXIT:
|
||||
LOG.info("Successfully killed the mitmproxy playback process"
|
||||
" with exit code %d" % exit_code)
|
||||
LOG.info(
|
||||
"Successfully killed the mitmproxy playback process"
|
||||
" with exit code %d" % exit_code
|
||||
)
|
||||
return
|
||||
log_func = LOG.error
|
||||
if self.ignore_mitmdump_exit_failure:
|
||||
|
|
@ -361,9 +390,8 @@ class MitmproxyDesktop(Mitmproxy):
|
|||
|
||||
self.write_policies_json(
|
||||
self.policies_dir,
|
||||
policies_content=POLICIES_CONTENT_ON % {"cert": self.cert_path,
|
||||
"host": self.host,
|
||||
"port": self.port},
|
||||
policies_content=POLICIES_CONTENT_ON
|
||||
% {"cert": self.cert_path, "host": self.host, "port": self.port},
|
||||
)
|
||||
|
||||
# cannot continue if failed to add CA cert to Firefox, need to check
|
||||
|
|
@ -396,8 +424,8 @@ class MitmproxyDesktop(Mitmproxy):
|
|||
LOG.info("Firefox policies file contents:")
|
||||
LOG.info(contents)
|
||||
if (
|
||||
POLICIES_CONTENT_ON
|
||||
% {"cert": self.cert_path, "host": self.host, "port": self.port}
|
||||
POLICIES_CONTENT_ON
|
||||
% {"cert": self.cert_path, "host": self.host, "port": self.port}
|
||||
) in contents:
|
||||
LOG.info("Verified mitmproxy CA certificate is installed in Firefox")
|
||||
else:
|
||||
|
|
@ -442,9 +470,12 @@ class MitmproxyAndroid(Mitmproxy):
|
|||
# when running locally, it is found in the Firefox desktop build (..obj../dist/bin)
|
||||
self.certutil_path = os.path.join(os.environ["MOZ_HOST_BIN"], "certutil")
|
||||
if not (
|
||||
os.path.isfile(self.certutil_path) and os.access(self.certutil_path, os.X_OK)
|
||||
os.path.isfile(self.certutil_path)
|
||||
and os.access(self.certutil_path, os.X_OK)
|
||||
):
|
||||
raise Exception("Abort: unable to execute certutil: {}".format(self.certutil_path))
|
||||
raise Exception(
|
||||
"Abort: unable to execute certutil: {}".format(self.certutil_path)
|
||||
)
|
||||
self.certutil_path = os.environ["MOZ_HOST_BIN"]
|
||||
os.environ["LD_LIBRARY_PATH"] = self.certutil_path
|
||||
else:
|
||||
|
|
@ -537,9 +568,7 @@ class MitmproxyAndroid(Mitmproxy):
|
|||
"-i",
|
||||
local_cert_path,
|
||||
]
|
||||
LOG.info(
|
||||
"importing mitmproxy cert into db using command"
|
||||
)
|
||||
LOG.info("importing mitmproxy cert into db using command")
|
||||
self.certutil(args)
|
||||
|
||||
def create_cert_db(self, cert_db_location):
|
||||
|
|
@ -593,7 +622,9 @@ class MitmproxyAndroid(Mitmproxy):
|
|||
cmd = [self.certutil_path] + list(args)
|
||||
LOG.info("Certutil: Running command: %s" % " ".join(cmd))
|
||||
try:
|
||||
cmd_proc = subprocess.Popen(cmd, stdout=PIPE, stderr=PIPE, env=os.environ.copy())
|
||||
cmd_proc = subprocess.Popen(
|
||||
cmd, stdout=PIPE, stderr=PIPE, env=os.environ.copy()
|
||||
)
|
||||
|
||||
cmd_output, errs = cmd_proc.communicate()
|
||||
except subprocess.SubprocessError:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,10 @@
|
|||
[
|
||||
{
|
||||
"size": 76345093,
|
||||
"visibility": "public",
|
||||
"digest": "4d653c0c74a8677e8e78cd72d109b1b54c75ef57b2e2ce980c5cfd602966c310065cf0e95c35f4fbfb1fe817f062ac6cf9cc129d72d42184b0237fb9b0bde081",
|
||||
"algorithm": "sha512",
|
||||
"filename": "mitmproxy-5.0.1-linux.tar.gz",
|
||||
"unpack": true
|
||||
}
|
||||
]
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
[
|
||||
{
|
||||
"size": 41341221,
|
||||
"visibility": "public",
|
||||
"digest": "4624bc26638cd7f3ab8c8d2ee8ab44ab81018114714a2b82614883adbfad540de73a97455fb228a3003006b4d03a8a9a597d70f8b27ccf1718f78380f911bdd8",
|
||||
"algorithm": "sha512",
|
||||
"filename": "mitmproxy-5.0.1-osx.tar.gz",
|
||||
"unpack": true
|
||||
}
|
||||
]
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
[
|
||||
{
|
||||
"size": 37875625,
|
||||
"visibility": "public",
|
||||
"digest": "d66234c9ca692d03412dd194b3f47c098e8ee1b17b178034fc86e0d8ada4d4f6cd1fbcfb62b7e7016539a878b1274ef83451a0acaca7011efaacb291fa52918d",
|
||||
"algorithm": "sha512",
|
||||
"filename": "mitmproxy-5.0.1-windows.zip",
|
||||
"unpack": true
|
||||
}
|
||||
]
|
||||
|
|
@ -85,6 +85,7 @@ Http2Layer._handle_remote_settings_changed = _remote_settings_changed
|
|||
|
||||
class AlternateServerPlayback:
|
||||
def __init__(self):
|
||||
|
||||
ctx.master.addons.remove(ctx.master.addons.get("serverplayback"))
|
||||
self.flowmap = {}
|
||||
self.configured = False
|
||||
|
|
@ -93,6 +94,9 @@ class AlternateServerPlayback:
|
|||
self._done = False
|
||||
self._replayed = 0
|
||||
self._not_replayed = 0
|
||||
self.mitm_version = ctx.mitmproxy.version.MITMPROXY
|
||||
|
||||
ctx.log.info("MitmProxy version: %s" % self.mitm_version)
|
||||
|
||||
def load(self, loader):
|
||||
loader.add_option(
|
||||
|
|
@ -112,16 +116,30 @@ class AlternateServerPlayback:
|
|||
"""
|
||||
for i in flows:
|
||||
if i.type == 'websocket':
|
||||
# Mitmproxy can't replay WebSocket packages.
|
||||
ctx.log.info(
|
||||
"Request is a WebSocketFlow. Removing from request list as WebSockets"
|
||||
" are dissabled "
|
||||
"Recorded response is a WebSocketFlow. Removing from recording list as"
|
||||
" WebSockets are disabled"
|
||||
)
|
||||
elif i.response:
|
||||
l = self.flowmap.setdefault(self._hash(i), [])
|
||||
l.append(i)
|
||||
# check if recorded request has a response associated
|
||||
if self.mitm_version == "5.0.1":
|
||||
if i.response.content:
|
||||
# Mitmproxy 5.0.1 Cannot assemble flow with missing content
|
||||
|
||||
l = self.flowmap.setdefault(self._hash(i), [])
|
||||
l.append(i)
|
||||
else:
|
||||
ctx.log.info(
|
||||
"Recorded response %s has no content. Removing from recording list"
|
||||
% i.request.url
|
||||
)
|
||||
if self.mitm_version == "4.0.4":
|
||||
l = self.flowmap.setdefault(self._hash(i), [])
|
||||
l.append(i)
|
||||
else:
|
||||
ctx.log.info(
|
||||
"Request %s has no response. Removing from request list"
|
||||
"Recorded request %s has no response. Removing from recording list"
|
||||
% i.request.url
|
||||
)
|
||||
ctx.master.addons.trigger("update", [])
|
||||
Загрузка…
Ссылка в новой задаче