Bug 1614565 - [Mozproxy] Add mitmproxy 5.0.1 capability to mozproxy r=tarek,perftest-reviewers,AlexandruIonescu

Differential Revision: https://phabricator.services.mozilla.com/D62420

--HG--
rename : testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay-4.0.4.py => testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay.py
extra : moz-landing-system : lando

testing/mozbase/mozproxy/mozproxy/backends/mitm/mitm.py

@@ -84,7 +84,9 @@ POLICIES_CONTENT_OFF = """{
 class Mitmproxy(Playback):
     def __init__(self, config):
         self.config = config
-        self.host = "127.0.0.1" if 'localhost' in self.config["host"] else self.config["host"]
+        self.host = (
+            "127.0.0.1" if "localhost" in self.config["host"] else self.config["host"]
+        )
         self.port = None
         self.mitmproxy_proc = None
         self.mitmdump_path = None
@@ -95,14 +97,21 @@ class Mitmproxy(Playback):
         )
 
         if self.config.get("playback_version") is None:
-            LOG.info("mitmproxy was not provided with a 'playback_version' "
+            LOG.info(
-                     "Using default playback version: 4.0.4")
+                "mitmproxy was not provided with a 'playback_version' "
+                "Using default playback version: 4.0.4"
+            )
             self.config["playback_version"] = "4.0.4"
 
         if self.config.get("playback_binary_manifest") is None:
-            LOG.info("mitmproxy was not provided with a 'playback_binary_manifest' "
+            LOG.info(
-                     "Using default playback_binary_manifest")
+                "mitmproxy was not provided with a 'playback_binary_manifest' "
-            self.config["playback_binary_manifest"] = "mitmproxy-rel-bin-4.0.4-{platform}.manifest"
+                "Using default playback_binary_manifest"
+            )
+            self.config["playback_binary_manifest"] = (
+                "mitmproxy-rel-bin-%s-{platform}.manifest"
+                % self.config["playback_version"]
+            )
 
         # mozproxy_dir is where we will download all mitmproxy required files
         # when running locally it comes from obj_path via mozharness/mach
@@ -127,6 +136,9 @@ class Mitmproxy(Playback):
         # where to get the data
         os.environ["MOZPROXY_DIR"] = self.mozproxy_dir
 
+        LOG.info("Playback tool: %s" % self.config["playback_tool"])
+        LOG.info("Playback tool version: %s" % self.config["playback_version"])
+
     def start(self):
         # go ahead and download and setup mitmproxy
         self.download()
@@ -151,9 +163,12 @@ class Mitmproxy(Playback):
 
         # generate the mitmdump_path
         self.mitmdump_path = os.path.normpath(
-                os.path.join(self.mozproxy_dir, "mitmdump-%s" %
+            os.path.join(
-                             self.config["playback_version"],
+                self.mozproxy_dir,
-                             "mitmdump"))
+                "mitmdump-%s" % self.config["playback_version"],
+                "mitmdump",
+            )
+        )
 
         # Check if mitmproxy bin exists
         if os.path.exists(self.mitmdump_path):
@@ -167,8 +182,8 @@ class Mitmproxy(Playback):
 
             LOG.info("downloading mitmproxy binary")
             tooltool_download(
-                transformed_manifest, self.config["run_local"],
+                transformed_manifest, self.config["run_local"], download_path
-                download_path)
+            )
 
         if "playback_pageset_manifest" in self.config:
             # we use one pageset for all platforms
@@ -213,44 +228,55 @@ class Mitmproxy(Playback):
         command = [mitmdump_path]
 
         # add proxy host and port options
-        command.extend(["--listen-host", self.host,
+        command.extend(["--listen-host", self.host, "--listen-port", str(self.port)])
-                        "--listen-port", str(self.port)])
 
         if "playback_tool_args" in self.config:
             LOG.info("Staring Proxy using provided command line!")
             command.extend(self.config["playback_tool_args"])
         elif "playback_files" in self.config:
             script = os.path.join(
-                os.path.dirname(os.path.realpath(__file__)), "scripts",
+                os.path.dirname(os.path.realpath(__file__)),
-                "alternate-server-replay-{}.py".format(
+                "scripts",
-                    self.config["playback_version"]))
+                "alternate-server-replay.py",
+            )
+            recording_paths = [
+                normalize_path(recording_path)
+                for recording_path in self.config["playback_files"]
+            ]
 
-            recording_paths = [normalize_path(recording_path)
+            if self.config["playback_version"] in ["4.0.4", "5.0.1"]:
-                               for recording_path in self.config["playback_files"]]
-
-            if self.config["playback_version"] == "4.0.4":
                 args = [
                     "-v",
-                    "--set", "upstream_cert=false",
+                    "--set",
-                    "--set", "upload_dir=" + normalize_path(self.upload_dir),
+                    "upstream_cert=false",
-                    "--set", "websocket=false",
+                    "--set",
-                    "--set", "server_replay_files={}".format(",".join(recording_paths)),
+                    "upload_dir=" + normalize_path(self.upload_dir),
-                    "--scripts", normalize_path(script),
+                    "--set",
+                    "websocket=false",
+                    "--set",
+                    "server_replay_files={}".format(",".join(recording_paths)),
+                    "--scripts",
+                    normalize_path(script),
                 ]
                 command.extend(args)
             else:
                 raise Exception("Mitmproxy version is unknown!")
 
         else:
-            raise Exception("Mitmproxy can't start playback! Playback settings missing.")
+            raise Exception(
+                "Mitmproxy can't start playback! Playback settings missing."
+            )
 
         LOG.info("Starting mitmproxy playback using env path: %s" % env["PATH"])
         LOG.info("Starting mitmproxy playback using command: %s" % " ".join(command))
         # to turn off mitmproxy log output, use these params for Popen:
         # Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
         self.mitmproxy_proc = ProcessHandler(
-            command, logfile=os.path.join(self.upload_dir, "mitmproxy.log"),
+            command,
-            env=env, processStderrLine=LOG.error, storeOutput=False
+            logfile=os.path.join(self.upload_dir, "mitmproxy.log"),
+            env=env,
+            processStderrLine=LOG.error,
+            storeOutput=False,
         )
         self.mitmproxy_proc.run()
         end_time = time.time() + MITMDUMP_COMMAND_TIMEOUT
@@ -295,9 +321,12 @@ class Mitmproxy(Playback):
 
             if mozinfo.os == "win":
                 from mozprocess.winprocess import ERROR_CONTROL_C_EXIT  # noqa
+
                 if exit_code == ERROR_CONTROL_C_EXIT:
-                    LOG.info("Successfully killed the mitmproxy playback process"
+                    LOG.info(
-                             " with exit code %d" % exit_code)
+                        "Successfully killed the mitmproxy playback process"
+                        " with exit code %d" % exit_code
+                    )
                     return
             log_func = LOG.error
             if self.ignore_mitmdump_exit_failure:
@@ -361,9 +390,8 @@ class MitmproxyDesktop(Mitmproxy):
 
         self.write_policies_json(
             self.policies_dir,
-            policies_content=POLICIES_CONTENT_ON % {"cert": self.cert_path,
+            policies_content=POLICIES_CONTENT_ON
-                                                    "host": self.host,
+            % {"cert": self.cert_path, "host": self.host, "port": self.port},
-                                                    "port": self.port},
         )
 
         # cannot continue if failed to add CA cert to Firefox, need to check
@@ -396,8 +424,8 @@ class MitmproxyDesktop(Mitmproxy):
             LOG.info("Firefox policies file contents:")
             LOG.info(contents)
             if (
-                    POLICIES_CONTENT_ON
+                POLICIES_CONTENT_ON
-                    % {"cert": self.cert_path, "host": self.host, "port": self.port}
+                % {"cert": self.cert_path, "host": self.host, "port": self.port}
             ) in contents:
                 LOG.info("Verified mitmproxy CA certificate is installed in Firefox")
             else:
@@ -442,9 +470,12 @@ class MitmproxyAndroid(Mitmproxy):
             # when running locally, it is found in the Firefox desktop build (..obj../dist/bin)
             self.certutil_path = os.path.join(os.environ["MOZ_HOST_BIN"], "certutil")
             if not (
-                    os.path.isfile(self.certutil_path) and os.access(self.certutil_path, os.X_OK)
+                os.path.isfile(self.certutil_path)
+                and os.access(self.certutil_path, os.X_OK)
             ):
-                raise Exception("Abort: unable to execute certutil: {}".format(self.certutil_path))
+                raise Exception(
+                    "Abort: unable to execute certutil: {}".format(self.certutil_path)
+                )
             self.certutil_path = os.environ["MOZ_HOST_BIN"]
             os.environ["LD_LIBRARY_PATH"] = self.certutil_path
         else:
@@ -537,9 +568,7 @@ class MitmproxyAndroid(Mitmproxy):
             "-i",
             local_cert_path,
         ]
-        LOG.info(
+        LOG.info("importing mitmproxy cert into db using command")
-            "importing mitmproxy cert into db using command"
-        )
         self.certutil(args)
 
     def create_cert_db(self, cert_db_location):
@@ -593,7 +622,9 @@ class MitmproxyAndroid(Mitmproxy):
         cmd = [self.certutil_path] + list(args)
         LOG.info("Certutil: Running command: %s" % " ".join(cmd))
         try:
-            cmd_proc = subprocess.Popen(cmd, stdout=PIPE, stderr=PIPE, env=os.environ.copy())
+            cmd_proc = subprocess.Popen(
+                cmd, stdout=PIPE, stderr=PIPE, env=os.environ.copy()
+            )
 
             cmd_output, errs = cmd_proc.communicate()
         except subprocess.SubprocessError:

testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-5.0.1-linux64.manifest

@@ -0,0 +1,10 @@
+[
+    {
+    "size": 76345093,
+    "visibility": "public",
+    "digest": "4d653c0c74a8677e8e78cd72d109b1b54c75ef57b2e2ce980c5cfd602966c310065cf0e95c35f4fbfb1fe817f062ac6cf9cc129d72d42184b0237fb9b0bde081",
+    "algorithm": "sha512",
+    "filename": "mitmproxy-5.0.1-linux.tar.gz",
+    "unpack": true
+    }
+]

testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-5.0.1-osx.manifest

@@ -0,0 +1,10 @@
+[
+  {
+    "size": 41341221,
+    "visibility": "public",
+    "digest": "4624bc26638cd7f3ab8c8d2ee8ab44ab81018114714a2b82614883adbfad540de73a97455fb228a3003006b4d03a8a9a597d70f8b27ccf1718f78380f911bdd8",
+    "algorithm": "sha512",
+    "filename": "mitmproxy-5.0.1-osx.tar.gz",
+    "unpack": true
+  }
+]

testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-5.0.1-win.manifest

@@ -0,0 +1,10 @@
+[
+  {
+    "size": 37875625,
+    "visibility": "public",
+    "digest": "d66234c9ca692d03412dd194b3f47c098e8ee1b17b178034fc86e0d8ada4d4f6cd1fbcfb62b7e7016539a878b1274ef83451a0acaca7011efaacb291fa52918d",
+    "algorithm": "sha512",
+    "filename": "mitmproxy-5.0.1-windows.zip",
+    "unpack": true
+  }
+]

testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay.py

@@ -85,6 +85,7 @@ Http2Layer._handle_remote_settings_changed = _remote_settings_changed
 
 class AlternateServerPlayback:
     def __init__(self):
+
         ctx.master.addons.remove(ctx.master.addons.get("serverplayback"))
         self.flowmap = {}
         self.configured = False
@@ -93,6 +94,9 @@ class AlternateServerPlayback:
         self._done = False
         self._replayed = 0
         self._not_replayed = 0
+        self.mitm_version = ctx.mitmproxy.version.MITMPROXY
+
+        ctx.log.info("MitmProxy version: %s" % self.mitm_version)
 
     def load(self, loader):
         loader.add_option(
@@ -112,16 +116,30 @@ class AlternateServerPlayback:
         """
         for i in flows:
             if i.type == 'websocket':
+                # Mitmproxy can't replay WebSocket packages.
                 ctx.log.info(
-                    "Request is a WebSocketFlow. Removing from request list as WebSockets"
+                    "Recorded response is a WebSocketFlow. Removing from recording list as"
-                    " are dissabled "
+                    "  WebSockets are disabled"
                 )
             elif i.response:
-                l = self.flowmap.setdefault(self._hash(i), [])
+                # check if recorded request has a response associated
-                l.append(i)
+                if self.mitm_version == "5.0.1":
+                    if i.response.content:
+                        # Mitmproxy 5.0.1 Cannot assemble flow with missing content
+
+                        l = self.flowmap.setdefault(self._hash(i), [])
+                        l.append(i)
+                    else:
+                        ctx.log.info(
+                             "Recorded response %s has no content. Removing from recording list"
+                             % i.request.url
+                        )
+                if self.mitm_version == "4.0.4":
+                    l = self.flowmap.setdefault(self._hash(i), [])
+                    l.append(i)
             else:
                 ctx.log.info(
-                    "Request %s has no response. Removing from request list"
+                    "Recorded request %s has no response. Removing from recording list"
                     % i.request.url
                 )
         ctx.master.addons.trigger("update", [])