diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitm.py b/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitm.py index 6e48d1807e10..11a74e30e2c6 100644 --- a/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitm.py +++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitm.py @@ -84,7 +84,9 @@ POLICIES_CONTENT_OFF = """{ class Mitmproxy(Playback): def __init__(self, config): self.config = config - self.host = "127.0.0.1" if 'localhost' in self.config["host"] else self.config["host"] + self.host = ( + "127.0.0.1" if "localhost" in self.config["host"] else self.config["host"] + ) self.port = None self.mitmproxy_proc = None self.mitmdump_path = None @@ -95,14 +97,21 @@ class Mitmproxy(Playback): ) if self.config.get("playback_version") is None: - LOG.info("mitmproxy was not provided with a 'playback_version' " - "Using default playback version: 4.0.4") + LOG.info( + "mitmproxy was not provided with a 'playback_version' " + "Using default playback version: 4.0.4" + ) self.config["playback_version"] = "4.0.4" if self.config.get("playback_binary_manifest") is None: - LOG.info("mitmproxy was not provided with a 'playback_binary_manifest' " - "Using default playback_binary_manifest") - self.config["playback_binary_manifest"] = "mitmproxy-rel-bin-4.0.4-{platform}.manifest" + LOG.info( + "mitmproxy was not provided with a 'playback_binary_manifest' " + "Using default playback_binary_manifest" + ) + self.config["playback_binary_manifest"] = ( + "mitmproxy-rel-bin-%s-{platform}.manifest" + % self.config["playback_version"] + ) # mozproxy_dir is where we will download all mitmproxy required files # when running locally it comes from obj_path via mozharness/mach @@ -127,6 +136,9 @@ class Mitmproxy(Playback): # where to get the data os.environ["MOZPROXY_DIR"] = self.mozproxy_dir + LOG.info("Playback tool: %s" % self.config["playback_tool"]) + LOG.info("Playback tool version: %s" % self.config["playback_version"]) + def start(self): # go ahead and download and setup mitmproxy self.download() @@ -151,9 +163,12 @@ class Mitmproxy(Playback): # generate the mitmdump_path self.mitmdump_path = os.path.normpath( - os.path.join(self.mozproxy_dir, "mitmdump-%s" % - self.config["playback_version"], - "mitmdump")) + os.path.join( + self.mozproxy_dir, + "mitmdump-%s" % self.config["playback_version"], + "mitmdump", + ) + ) # Check if mitmproxy bin exists if os.path.exists(self.mitmdump_path): @@ -167,8 +182,8 @@ class Mitmproxy(Playback): LOG.info("downloading mitmproxy binary") tooltool_download( - transformed_manifest, self.config["run_local"], - download_path) + transformed_manifest, self.config["run_local"], download_path + ) if "playback_pageset_manifest" in self.config: # we use one pageset for all platforms @@ -213,44 +228,55 @@ class Mitmproxy(Playback): command = [mitmdump_path] # add proxy host and port options - command.extend(["--listen-host", self.host, - "--listen-port", str(self.port)]) + command.extend(["--listen-host", self.host, "--listen-port", str(self.port)]) if "playback_tool_args" in self.config: LOG.info("Staring Proxy using provided command line!") command.extend(self.config["playback_tool_args"]) elif "playback_files" in self.config: script = os.path.join( - os.path.dirname(os.path.realpath(__file__)), "scripts", - "alternate-server-replay-{}.py".format( - self.config["playback_version"])) + os.path.dirname(os.path.realpath(__file__)), + "scripts", + "alternate-server-replay.py", + ) + recording_paths = [ + normalize_path(recording_path) + for recording_path in self.config["playback_files"] + ] - recording_paths = [normalize_path(recording_path) - for recording_path in self.config["playback_files"]] - - if self.config["playback_version"] == "4.0.4": + if self.config["playback_version"] in ["4.0.4", "5.0.1"]: args = [ "-v", - "--set", "upstream_cert=false", - "--set", "upload_dir=" + normalize_path(self.upload_dir), - "--set", "websocket=false", - "--set", "server_replay_files={}".format(",".join(recording_paths)), - "--scripts", normalize_path(script), + "--set", + "upstream_cert=false", + "--set", + "upload_dir=" + normalize_path(self.upload_dir), + "--set", + "websocket=false", + "--set", + "server_replay_files={}".format(",".join(recording_paths)), + "--scripts", + normalize_path(script), ] command.extend(args) else: raise Exception("Mitmproxy version is unknown!") else: - raise Exception("Mitmproxy can't start playback! Playback settings missing.") + raise Exception( + "Mitmproxy can't start playback! Playback settings missing." + ) LOG.info("Starting mitmproxy playback using env path: %s" % env["PATH"]) LOG.info("Starting mitmproxy playback using command: %s" % " ".join(command)) # to turn off mitmproxy log output, use these params for Popen: # Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) self.mitmproxy_proc = ProcessHandler( - command, logfile=os.path.join(self.upload_dir, "mitmproxy.log"), - env=env, processStderrLine=LOG.error, storeOutput=False + command, + logfile=os.path.join(self.upload_dir, "mitmproxy.log"), + env=env, + processStderrLine=LOG.error, + storeOutput=False, ) self.mitmproxy_proc.run() end_time = time.time() + MITMDUMP_COMMAND_TIMEOUT @@ -295,9 +321,12 @@ class Mitmproxy(Playback): if mozinfo.os == "win": from mozprocess.winprocess import ERROR_CONTROL_C_EXIT # noqa + if exit_code == ERROR_CONTROL_C_EXIT: - LOG.info("Successfully killed the mitmproxy playback process" - " with exit code %d" % exit_code) + LOG.info( + "Successfully killed the mitmproxy playback process" + " with exit code %d" % exit_code + ) return log_func = LOG.error if self.ignore_mitmdump_exit_failure: @@ -361,9 +390,8 @@ class MitmproxyDesktop(Mitmproxy): self.write_policies_json( self.policies_dir, - policies_content=POLICIES_CONTENT_ON % {"cert": self.cert_path, - "host": self.host, - "port": self.port}, + policies_content=POLICIES_CONTENT_ON + % {"cert": self.cert_path, "host": self.host, "port": self.port}, ) # cannot continue if failed to add CA cert to Firefox, need to check @@ -396,8 +424,8 @@ class MitmproxyDesktop(Mitmproxy): LOG.info("Firefox policies file contents:") LOG.info(contents) if ( - POLICIES_CONTENT_ON - % {"cert": self.cert_path, "host": self.host, "port": self.port} + POLICIES_CONTENT_ON + % {"cert": self.cert_path, "host": self.host, "port": self.port} ) in contents: LOG.info("Verified mitmproxy CA certificate is installed in Firefox") else: @@ -442,9 +470,12 @@ class MitmproxyAndroid(Mitmproxy): # when running locally, it is found in the Firefox desktop build (..obj../dist/bin) self.certutil_path = os.path.join(os.environ["MOZ_HOST_BIN"], "certutil") if not ( - os.path.isfile(self.certutil_path) and os.access(self.certutil_path, os.X_OK) + os.path.isfile(self.certutil_path) + and os.access(self.certutil_path, os.X_OK) ): - raise Exception("Abort: unable to execute certutil: {}".format(self.certutil_path)) + raise Exception( + "Abort: unable to execute certutil: {}".format(self.certutil_path) + ) self.certutil_path = os.environ["MOZ_HOST_BIN"] os.environ["LD_LIBRARY_PATH"] = self.certutil_path else: @@ -537,9 +568,7 @@ class MitmproxyAndroid(Mitmproxy): "-i", local_cert_path, ] - LOG.info( - "importing mitmproxy cert into db using command" - ) + LOG.info("importing mitmproxy cert into db using command") self.certutil(args) def create_cert_db(self, cert_db_location): @@ -593,7 +622,9 @@ class MitmproxyAndroid(Mitmproxy): cmd = [self.certutil_path] + list(args) LOG.info("Certutil: Running command: %s" % " ".join(cmd)) try: - cmd_proc = subprocess.Popen(cmd, stdout=PIPE, stderr=PIPE, env=os.environ.copy()) + cmd_proc = subprocess.Popen( + cmd, stdout=PIPE, stderr=PIPE, env=os.environ.copy() + ) cmd_output, errs = cmd_proc.communicate() except subprocess.SubprocessError: diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-5.0.1-linux64.manifest b/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-5.0.1-linux64.manifest new file mode 100644 index 000000000000..cfbbd667fd6b --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-5.0.1-linux64.manifest @@ -0,0 +1,10 @@ +[ + { + "size": 76345093, + "visibility": "public", + "digest": "4d653c0c74a8677e8e78cd72d109b1b54c75ef57b2e2ce980c5cfd602966c310065cf0e95c35f4fbfb1fe817f062ac6cf9cc129d72d42184b0237fb9b0bde081", + "algorithm": "sha512", + "filename": "mitmproxy-5.0.1-linux.tar.gz", + "unpack": true + } +] diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-5.0.1-osx.manifest b/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-5.0.1-osx.manifest new file mode 100644 index 000000000000..2a97c84eed6f --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-5.0.1-osx.manifest @@ -0,0 +1,10 @@ +[ + { + "size": 41341221, + "visibility": "public", + "digest": "4624bc26638cd7f3ab8c8d2ee8ab44ab81018114714a2b82614883adbfad540de73a97455fb228a3003006b4d03a8a9a597d70f8b27ccf1718f78380f911bdd8", + "algorithm": "sha512", + "filename": "mitmproxy-5.0.1-osx.tar.gz", + "unpack": true + } +] diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-5.0.1-win.manifest b/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-5.0.1-win.manifest new file mode 100644 index 000000000000..2d7d60715a3b --- /dev/null +++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-5.0.1-win.manifest @@ -0,0 +1,10 @@ +[ + { + "size": 37875625, + "visibility": "public", + "digest": "d66234c9ca692d03412dd194b3f47c098e8ee1b17b178034fc86e0d8ada4d4f6cd1fbcfb62b7e7016539a878b1274ef83451a0acaca7011efaacb291fa52918d", + "algorithm": "sha512", + "filename": "mitmproxy-5.0.1-windows.zip", + "unpack": true + } +] diff --git a/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay-4.0.4.py b/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay.py similarity index 87% rename from testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay-4.0.4.py rename to testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay.py index 950917002cbd..492a1fb05edd 100644 --- a/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay-4.0.4.py +++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay.py @@ -85,6 +85,7 @@ Http2Layer._handle_remote_settings_changed = _remote_settings_changed class AlternateServerPlayback: def __init__(self): + ctx.master.addons.remove(ctx.master.addons.get("serverplayback")) self.flowmap = {} self.configured = False @@ -93,6 +94,9 @@ class AlternateServerPlayback: self._done = False self._replayed = 0 self._not_replayed = 0 + self.mitm_version = ctx.mitmproxy.version.MITMPROXY + + ctx.log.info("MitmProxy version: %s" % self.mitm_version) def load(self, loader): loader.add_option( @@ -112,16 +116,30 @@ class AlternateServerPlayback: """ for i in flows: if i.type == 'websocket': + # Mitmproxy can't replay WebSocket packages. ctx.log.info( - "Request is a WebSocketFlow. Removing from request list as WebSockets" - " are dissabled " + "Recorded response is a WebSocketFlow. Removing from recording list as" + " WebSockets are disabled" ) elif i.response: - l = self.flowmap.setdefault(self._hash(i), []) - l.append(i) + # check if recorded request has a response associated + if self.mitm_version == "5.0.1": + if i.response.content: + # Mitmproxy 5.0.1 Cannot assemble flow with missing content + + l = self.flowmap.setdefault(self._hash(i), []) + l.append(i) + else: + ctx.log.info( + "Recorded response %s has no content. Removing from recording list" + % i.request.url + ) + if self.mitm_version == "4.0.4": + l = self.flowmap.setdefault(self._hash(i), []) + l.append(i) else: ctx.log.info( - "Request %s has no response. Removing from request list" + "Recorded request %s has no response. Removing from recording list" % i.request.url ) ctx.master.addons.trigger("update", [])