Bug 1643263 Update mitm from 5.0.1 to 5.1.1. r=perftest-reviewers,Bebe

This patch is updating the code to mitm 5.1.1 and mitmproxy manifest.

Differential Revision: https://phabricator.services.mozilla.com/D79256

testing/mozbase/mozproxy/mozproxy/backends/mitm/mitm.py

@@ -247,7 +247,7 @@ class Mitmproxy(Playback):
             self.recording_paths = [normalize_path(recording_path)
                                     for recording_path in self.config["playback_files"]]
 
-            if self.config["playback_version"] in ["4.0.4", "5.0.1"]:
+            if self.config["playback_version"] in ["4.0.4", "5.1.1"]:
                 args = [
                     "-v",
                     "--set",

testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-5.1.1-linux64.manifest

@@ -0,0 +1,10 @@
+[
+    {
+    "size": 24989250,
+    "visibility": "public",
+    "digest": "75ea9d024cc9138e5bc993c292ec65c2a40ce4e382f01c41ef865ddfac6f1553ee0e73e3ff2003234d6ecab2d3cdbbd38809cc7e62bbcabaed521a623575e2b8",
+    "algorithm": "sha512",
+    "filename": "mitmproxy-5.1.1-linux.tar.gz",
+    "unpack": true
+    }
+]

testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-5.1.1-osx.manifest

@@ -0,0 +1,10 @@
+[
+  {
+    "size": 13178683,
+    "visibility": "public",
+    "digest": "efbc90674a85165e9065e70a8f0c2d4e42688a045130546c2d5721cdb4c54637f7d2b9f4e6a6953a37f94d0edb5bf128c25d7a628246a75d4ad7ba6c884589a9",
+    "algorithm": "sha512",
+    "filename": "mitmproxy-5.1.1-osx.tar.gz",
+    "unpack": true
+  }
+]

testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-5.1.1-win.manifest

@@ -0,0 +1,10 @@
+[
+  {
+    "size": 18019253,
+    "visibility": "public",
+    "digest": "69470f6a58c50912072ab6a911ac3e266d5ec8c9410c8aa4bad76cd7f61bca640e748fd682379ef1523f12da2a8c7a9c67d5bbae5f6d6fa164c2c6b9765b79c1",
+    "algorithm": "sha512",
+    "filename": "mitmproxy-5.1.1-windows.zip",
+    "unpack": true
+  }
+]

testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay.py

@@ -120,23 +120,10 @@ class AlternateServerPlayback:
                     "Recorded response is a WebSocketFlow. Removing from recording list as"
                     "  WebSockets are disabled"
                 )
-            elif i.response:
-                # check if recorded request has a response associated
-                if self.mitm_version == "5.0.1":
-                    if i.response.content:
-                        # Mitmproxy 5.0.1 Cannot assemble flow with missing content
-
-                        l = self.flowmap.setdefault(self._hash(i), [])
-                        l.append(i)
-                    else:
-                        ctx.log.info(
-                             "Recorded response %s has no content. Removing from recording list"
-                             % i.request.url
-                        )
-                if self.mitm_version in ("4.0.2", "4.0.4"):
-                    # see: https://github.com/mitmproxy/mitmproxy/issues/3856
-                    l = self.flowmap.setdefault(self._hash(i), [])
-                    l.append(i)
+            elif i.response and self.mitm_version in ("4.0.2", "4.0.4", "5.1.1"):
+                # see: https://github.com/mitmproxy/mitmproxy/issues/3856
+                l = self.flowmap.setdefault(self._hash(i), [])
+                l.append(i)
             else:
                 ctx.log.info(
                     "Recorded request %s has no response. Removing from recording list"
@@ -150,6 +137,8 @@ class AlternateServerPlayback:
                 paths = paths[0].split(",")
             for path in paths:
                 ctx.log.info("Loading flows from %s" % path)
+                if not os.path.exists(path):
+                    raise Exception("File does not exist!")
                 try:
                     flows = io.read_flows_from_paths([path])
                 except exceptions.FlowReadException as e:

testing/mozbase/mozproxy/tests/test_proxy.py

@@ -64,7 +64,7 @@ def get_status_code(url, playback):
 
 def test_mitm_check_proxy(*args):
     # test setup
-    bin_name = "mitmproxy-rel-bin-4.0.4-{platform}.manifest"
+    bin_name = "mitmproxy-rel-bin-5.1.1-{platform}.manifest"
     pageset_name = "mitm4-linux-firefox-amazon.manifest"
     playback_recordings = "amazon.mp"
 
@@ -72,7 +72,7 @@ def test_mitm_check_proxy(*args):
         "playback_tool": "mitmproxy",
         "playback_binary_manifest": bin_name,
         "playback_pageset_manifest": os.path.join(here, "files", pageset_name),
-        "playback_version": '4.0.4',
+        "playback_version": '5.1.1',
         "platform": mozinfo.os,
         "run_local": "MOZ_AUTOMATION" not in os.environ,
         "binary": "firefox",
@@ -104,14 +104,14 @@ def test_mitm_check_proxy(*args):
 @mock.patch("mozproxy.utils.ProcessHandler", new=Process)
 @mock.patch("os.kill", new=kill)
 def test_mitm(*args):
-    bin_name = "mitmproxy-rel-bin-4.0.4-{platform}.manifest"
+    bin_name = "mitmproxy-rel-bin-5.1.1-{platform}.manifest"
     pageset_name = "mitm4-linux-firefox-amazon.manifest"
 
     config = {
         "playback_tool": "mitmproxy",
         "playback_binary_manifest": bin_name,
         "playback_pageset_manifest": pageset_name,
-        "playback_version": '4.0.4',
+        "playback_version": '5.1.1',
         "platform": mozinfo.os,
         "playback_recordings": os.path.join(here, "paypal.mp"),
         "run_local": True,
@@ -145,14 +145,14 @@ def test_playback_setup_failed(*args):
 
         return _s
 
-    bin_name = "mitmproxy-rel-bin-4.0.4-{platform}.manifest"
+    bin_name = "mitmproxy-rel-bin-5.1.1-{platform}.manifest"
     pageset_name = "mitm4-linux-firefox-amazon.manifest"
 
     config = {
         "playback_tool": "mitmproxy",
         "playback_binary_manifest": bin_name,
         "playback_pageset_manifest": pageset_name,
-        "playback_version": '4.0.4',
+        "playback_version": '5.1.1',
         "platform": mozinfo.os,
         "playback_recordings": os.path.join(here, "paypal.mp"),
         "run_local": True,
@@ -182,14 +182,14 @@ def test_playback_setup_failed(*args):
 @mock.patch("mozproxy.utils.ProcessHandler", new=ProcessWithRetry)
 @mock.patch("os.kill", new=kill)
 def test_mitm_with_retry(*args):
-    bin_name = "mitmproxy-rel-bin-4.0.4-{platform}.manifest"
+    bin_name = "mitmproxy-rel-bin-5.1.1-{platform}.manifest"
     pageset_name = "mitm4-linux-firefox-amazon.manifest"
 
     config = {
         "playback_tool": "mitmproxy",
         "playback_binary_manifest": bin_name,
         "playback_pageset_manifest": pageset_name,
-        "playback_version": '4.0.4',
+        "playback_version": '5.1.1',
         "platform": mozinfo.os,
         "playback_recordings": os.path.join(here, "paypal.mp"),
         "run_local": True,

testing/raptor/test/test_playback.py

@@ -31,8 +31,8 @@ def test_get_playback(get_binary):
         return
     config["obj_path"] = os.path.dirname(get_binary("firefox"))
     config["playback_tool"] = "mitmproxy"
-    config["playback_version"] = "4.0.4"
-    config["playback_binary_manifest"] = "mitmproxy-rel-bin-4.0.4-{platform}.manifest"
+    config["playback_version"] = "5.1.1"
+    config["playback_binary_manifest"] = "mitmproxy-rel-bin-5.1.1-{platform}.manifest"
     config["playback_pageset_manifest"] = os.path.join(
         os.path.dirname(os.path.abspath(os.path.dirname(__file__))),
         "raptor",