Bug 1539983 [mozproxy] Move alternate-server-replay script into mozproxy r=tarek

Differential Revision: https://phabricator.services.mozilla.com/D41540

--HG--
rename : testing/mozbase/mozproxy/mozproxy/backends/mitm.py => testing/mozbase/mozproxy/mozproxy/backends/mitm/mitm.py
rename : testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-4.0.4-linux64.manifest => testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-4.0.4-linux64.manifest
rename : testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-4.0.4-osx.manifest => testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-4.0.4-osx.manifest
rename : testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-4.0.4-win.manifest => testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-4.0.4-win.manifest
rename : testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-linux64.manifest => testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-linux64.manifest
rename : testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-osx.manifest => testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-osx.manifest
rename : testing/mozbase/mozproxy/mozproxy/backends/mitmproxy-rel-bin-win.manifest => testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy-rel-bin-win.manifest
rename : testing/mozbase/mozproxy/mozproxy/backends/mitmproxy_requirements.txt => testing/mozbase/mozproxy/mozproxy/backends/mitm/mitmproxy_requirements.txt
rename : testing/raptor/raptor/playback/alternate-server-replay-2.0.2.py => testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay-2.0.2.py
rename : testing/raptor/raptor/playback/alternate-server-replay-4.0.4.py => testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay-4.0.4.py
rename : testing/raptor/raptor/playback/scripts/catapult/LICENSE => testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/catapult/LICENSE
rename : testing/raptor/raptor/playback/scripts/catapult/deterministic.js => testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/catapult/deterministic.js
rename : testing/raptor/raptor/playback/scripts/inject-deterministic.py => testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/inject-deterministic.py
extra : moz-landing-system : lando

.eslintignore

@@ -295,6 +295,10 @@ testing/modules/ajv-4.1.1.js
 testing/modules/sinon-7.2.7.js
 # octothorpe used for pref file comment causes parsing error
 testing/mozbase/mozprofile/tests/files/prefs_with_comments.js
+
+# Mozproxy third party
+testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/catapult/**
+
 testing/talos/talos/scripts/jszip.min.js
 testing/talos/talos/startup_test/sessionrestore/profile/sessionstore.js
 testing/talos/talos/startup_test/sessionrestore/profile-manywindows/sessionstore.js
@@ -304,8 +308,6 @@ testing/talos/talos/tests/v8_7/**
 testing/talos/talos/tests/kraken/**
 # Runing Talos may extract data here, see bug 1435677.
 testing/talos/talos/tests/tp5n/**
-# Raptor third party
-testing/raptor/raptor/playback/scripts/catapult/**
 
 testing/web-platform/**
 testing/xpcshell/moz-http2/**

testing/mozbase/mozproxy/mozproxy/backends/mitm/__init__.py

@@ -0,0 +1,8 @@
+# flake8: noqa
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+from .mitm import *

testing/mozbase/mozproxy/mozproxy/backends/mitm/mitm.py

@@ -195,7 +195,51 @@ class Mitmproxy(Playback):
         command = [mitmdump_path]
 
         if "playback_tool_args" in self.config:
+            LOG.info("Staring Proxy using provided command line!")
             command.extend(self.config["playback_tool_args"])
+        elif "playback_files" in self.config:
+            script = os.path.join(
+                                os.path.dirname(os.path.realpath(__file__)), "scripts",
+                                "alternate-server-replay-{}.py".format(
+                                    self.config["playback_version"]))
+            recording_paths = self.config["playback_files"]
+            # this part is platform-specific
+            if mozinfo.os == "win":
+                script = script.replace("\\", "\\\\\\")
+                recording_paths = [recording_path.replace("\\", "\\\\\\")
+                                   for recording_path in recording_paths]
+
+            if self.config["playback_version"] == "2.0.2":
+                args = [
+                    "--replay-kill-extra",
+                    "-v",
+                    "--script",
+                    '""{} {}""'.format(script, " ".join(recording_paths)),
+                ]
+
+                if not self.config["playback_upstream_cert"]:
+                    LOG.info("No upstream certificate sniffing")
+                    args.insert(0, "--no-upstream-cert")
+                self.playback.config["playback_tool_args"] = args
+            elif self.config["playback_version"] == "4.0.4":
+                args = [
+                    "-v",
+                    "--set",
+                    "websocket=false",
+                    "--set",
+                    "server_replay_files={}".format(" ".join(recording_paths)),
+                    "--scripts",
+                    script,
+                ]
+                if not self.config["playback_upstream_cert"]:
+                    LOG.info("No upstream certificate sniffing")
+                    args = ["--set", "upstream_cert=false"] + args
+                command.extend(args)
+            else:
+                raise Exception("Mitmproxy version is unknown!")
+
+        else:
+            raise Exception("Mitmproxy can't start playback! Playback settings missing.")
 
         LOG.info("Starting mitmproxy playback using env path: %s" % env["PATH"])
         LOG.info("Starting mitmproxy playback using command: %s" % " ".join(command))

testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/alternate-server-replay-4.0.4.py

@@ -4,6 +4,7 @@
 # Altered features:
 # * returns 404 rather than dropping the whole HTTP/2 connection on the floor
 # * remove the replay packages that don't have any content in their response package
+from __future__ import absolute_import, print_function
 
 import os
 import json

testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/inject-deterministic.py

@@ -1,3 +1,5 @@
+from __future__ import absolute_import, print_function
+
 import base64
 import hashlib
 import re

testing/mozbase/mozproxy/tests/test_proxy.py

@@ -31,13 +31,15 @@ class Process:
 @mock.patch("mozproxy.backends.mitm.tooltool_download", new=mock.DEFAULT)
 @mock.patch("mozproxy.backends.mitm.Mitmproxy.check_proxy", lambda x: True)
 def test_mitm(*args):
-    bin_name = "mitmproxy-rel-bin-{platform}.manifest"
-    pageset_name = "mitmproxy-recordings-raptor-paypal.manifest"
+    bin_name = "mitmproxy-rel-bin-4.0.4-{platform}.manifest"
+    pageset_name = "mitm4-linux-firefox-amazon.manifest"
 
     config = {
         "playback_tool": "mitmproxy",
         "playback_binary_manifest": bin_name,
         "playback_pageset_manifest": pageset_name,
+        "playback_upstream_cert": 'false',
+        "playback_version": '4.0.4',
         "platform": mozinfo.os,
         "playback_recordings": os.path.join(here, "paypal.mp"),
         "run_local": True,
@@ -49,7 +51,7 @@ def test_mitm(*args):
     with tempdir() as obj_path:
         config["obj_path"] = obj_path
         playback = get_playback(config)
-
+        playback.config['playback_files'] = config['playback_recordings']
     assert playback is not None
     try:
         playback.start()
@@ -70,13 +72,15 @@ def test_playback_setup_failed(*args):
 
         return _s
 
-    bin_name = "mitmproxy-rel-bin-{platform}.manifest"
-    pageset_name = "mitmproxy-recordings-raptor-paypal.manifest"
+    bin_name = "mitmproxy-rel-bin-4.0.4-{platform}.manifest"
+    pageset_name = "mitm4-linux-firefox-amazon.manifest"
 
     config = {
         "playback_tool": "mitmproxy",
         "playback_binary_manifest": bin_name,
         "playback_pageset_manifest": pageset_name,
+        "playback_upstream_cert": 'false',
+        "playback_version": '4.0.4',
         "platform": mozinfo.os,
         "playback_recordings": os.path.join(here, "paypal.mp"),
         "run_local": True,
@@ -93,6 +97,7 @@ def test_playback_setup_failed(*args):
             with mock.patch(prefix + "stop_mitmproxy_playback") as p:
                 try:
                     pb = get_playback(config)
+                    pb.config['playback_files'] = config['playback_recordings']
                     pb.start()
                 except SetupFailed:
                     assert p.call_count == 1

testing/raptor/raptor/raptor.py

@@ -402,53 +402,12 @@ class Raptor(Perftest):
         if self.config['app'] in chrome_apps:
             self.profile.addons.remove(self.raptor_webext)
 
-    def get_proxy_command_for_mitm(self, test, version):
-        # Generate Mitmproxy playback args
-        script = os.path.join(here, "playback", "alternate-server-replay-{}.py".format(version))
-
-        recording_paths = self.get_recording_paths(test)
-
-        # this part is platform-specific
-        if mozinfo.os == "win":
-            script = script.replace("\\", "\\\\\\")
-            recording_paths = [recording_path.replace("\\", "\\\\\\")
-                               for recording_path in recording_paths]
-
-        if version == "2.0.2":
-            args = [
-                "--replay-kill-extra",
-                "-v",
-                "--script",
-                '""{} {}""'.format(script, " ".join(recording_paths)),
-            ]
-
-            if not self.config["playback_upstream_cert"]:
-                LOG.info("No upstream certificate sniffing")
-                args.insert(0, "--no-upstream-cert")
-            self.playback.config["playback_tool_args"] = args
-        elif version == "4.0.4":
-            args = [
-                "-v",
-                "--set",
-                "websocket=false",
-                "--set",
-                "server_replay_files={}".format(" ".join(recording_paths)),
-                "--scripts",
-                script,
-            ]
-            if not self.config["playback_upstream_cert"]:
-                LOG.info("No upstream certificate sniffing")
-                args = ["--set", "upstream_cert=false"] + args
-            self.playback.config["playback_tool_args"] = args
-        else:
-            raise Exception("Mitmproxy version is unknown!")
-
     def start_playback(self, test):
         # creating the playback tool
         self.get_playback_config(test)
         self.playback = get_playback(self.config, self.device)
 
-        self.get_proxy_command_for_mitm(test, self.config['playback_version'])
+        self.playback.config['playback_files'] = self.get_recording_paths(test)
 
         # let's start it!
         self.playback.start()

testing/raptor/test/test_playback.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import, unicode_literals
 
 import os
+import time
 
 import mozinfo
 import mozunit
@@ -27,17 +28,24 @@ def test_get_playback(get_binary):
         return
     config['obj_path'] = os.path.dirname(get_binary('firefox'))
     config['playback_tool'] = 'mitmproxy'
-    config['playback_binary_manifest'] = 'mitmproxy-rel-bin-osx.manifest'
-    config['playback_pageset_manifest'] = 'mitmproxy-recordings-raptor-tp6-1.manifest'
+    config['playback_version'] = '4.0.4'
+    config['playback_upstream_cert'] = 'false'
+    config['playback_binary_manifest'] = 'mitmproxy-rel-bin-4.0.4-{platform}.manifest'
+    config['playback_pageset_manifest'] = os.path.join(
+        os.path.dirname(os.path.abspath(os.path.dirname(__file__))), "raptor", "playback",
+        'mitm4-linux-firefox-amazon.manifest')
     config['playback_recordings'] = 'amazon.mp'
     config['binary'] = get_binary('firefox')
     config['run_local'] = run_local
     config['app'] = 'firefox'
-    config['host'] = 'example.com'
+    config['host'] = 'https://www.amazon.com/s?k=laptop&ref=nb_sb_noss_1'
 
     playback = get_playback(config)
+    playback.config['playback_files'] = [os.path.join(playback.mozproxy_dir,
+                                                      config['playback_recordings'])]
     assert isinstance(playback, MitmproxyDesktop)
     playback.start()
+    time.sleep(1)
     playback.stop()