Bug 1532491 - Part 2: Add a tp6 version of AWSY. r=bc,rwood

This adds a '--tp6' option to `mach awsy-test` which allows the running of the AWSY test against the tp6 pageset rather than the default tp5 pageset.

Differential Revision: https://phabricator.services.mozilla.com/D23606

--HG--
extra : moz-landing-system : lando

testing/awsy/awsy/alternate-server-replay.py

@@ -0,0 +1,186 @@
+# This file was copied from mitmproxy/addons/serverplayback.py release tag 2.0.2 and modified by
+# Benjamin Smedberg
+
+# Altered features:
+# * --kill returns 404 rather than dropping the whole HTTP/2 connection on the floor
+# * best-match response handling is used to improve success rates
+from __future__ import absolute_import, print_function
+
+import hashlib
+import sys
+import urllib
+from collections import defaultdict
+
+from mitmproxy import ctx
+from mitmproxy import exceptions
+from mitmproxy import http
+from mitmproxy import io
+from typing import Any  # noqa
+from typing import List  # noqa
+
+
+class ServerPlayback:
+    def __init__(self, replayfiles):
+        self.options = None
+        self.replayfiles = replayfiles
+        self.flowmap = {}
+
+    def load(self, flows):
+        for i in flows:
+            if i.response:
+                l = self.flowmap.setdefault(self._hash(i.request), [])
+                l.append(i)
+
+    def clear(self):
+        self.flowmap = {}
+
+    def _parse(self, r):
+        """
+            Return (path, queries, formdata, content) for a request.
+        """
+        _, _, path, _, query, _ = urllib.parse.urlparse(r.url)
+        queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True)
+        queries = defaultdict(list)
+        for k, v in queriesArray:
+            queries[k].append(v)
+
+        content = None
+        formdata = None
+        if r.raw_content != b'':
+            if r.multipart_form:
+                formdata = r.multipart_form
+            elif r.urlencoded_form:
+                formdata = r.urlencoded_form
+            else:
+                content = r.content
+        return (path, queries, formdata, content)
+
+    def _hash(self, r):
+        """
+            Calculates a loose hash of the flow request.
+        """
+        path, queries, _, _ = self._parse(r)
+
+        key = [str(r.port), str(r.scheme), str(r.method), str(path)]  # type: List[Any]
+        if not self.options.server_replay_ignore_host:
+            key.append(r.host)
+
+        if len(queries):
+            key.append("?")
+
+        return hashlib.sha256(
+            repr(key).encode("utf8", "surrogateescape")
+        ).digest()
+
+    def _match(self, request_a, request_b):
+        """
+            Calculate a match score between two requests.
+            Match algorithm:
+              * identical query keys: 3 points
+              * matching query param present: 1 point
+              * matching query param value: 3 points
+              * identical form keys: 3 points
+              * matching form param present: 1 point
+              * matching form param value: 3 points
+              * matching body (no multipart or encoded form): 4 points
+        """
+        match = 0
+
+        path_a, queries_a, form_a, content_a = self._parse(request_a)
+        path_b, queries_b, form_b, content_b = self._parse(request_b)
+
+        keys_a = set(queries_a.keys())
+        keys_b = set(queries_b.keys())
+        if keys_a == keys_b:
+            match += 3
+
+        for key in keys_a:
+            values_a = set(queries_a[key])
+            values_b = set(queries_b[key])
+            if len(values_a) == len(values_b):
+                match += 1
+            if values_a == values_b:
+                match += 3
+
+        if form_a and form_b:
+            keys_a = set(form_a.keys())
+            keys_b = set(form_b.keys())
+            if keys_a == keys_b:
+                match += 3
+
+            for key in keys_a:
+                values_a = set(form_a.get_all(key))
+                values_b = set(form_b.get_all(key))
+                if len(values_a) == len(values_b):
+                    match += 1
+                if values_a == values_b:
+                    match += 3
+
+        elif content_a and (content_a == content_b):
+            match += 4
+
+        return match
+
+    def next_flow(self, request):
+        """
+            Returns the next flow object, or None if no matching flow was
+            found.
+        """
+        hsh = self._hash(request)
+        flows = self.flowmap.get(hsh, None)
+        if flows is None:
+            return None
+
+        # if it's an exact match, great!
+        if len(flows) == 1:
+            candidate = flows[0]
+            if (candidate.request.url == request.url and
+               candidate.request.raw_content == request.raw_content):
+                ctx.log.info("For request {} found exact replay match".format(request.url))
+                return candidate
+
+        # find the best match between the request and the available flow candidates
+        match = -1
+        flow = None
+        ctx.log.debug("Candiate flows for request: {}".format(request.url))
+        for candidate_flow in flows:
+            candidate_match = self._match(candidate_flow.request, request)
+            ctx.log.debug("  score={} url={}".format(candidate_match, candidate_flow.request.url))
+            if candidate_match >= match:
+                match = candidate_match
+                flow = candidate_flow
+        ctx.log.info("For request {} best match {} with score=={}".format(request.url,
+                     flow.request.url, match))
+        return flow
+
+    def configure(self, options, updated):
+        self.options = options
+        self.clear()
+        try:
+            flows = io.read_flows_from_paths(self.replayfiles)
+        except exceptions.FlowReadException as e:
+            raise exceptions.OptionsError(str(e))
+        self.load(flows)
+
+    def request(self, f):
+        if self.flowmap:
+            rflow = self.next_flow(f.request)
+            if rflow:
+                response = rflow.response.copy()
+                response.is_replay = True
+                if self.options.refresh_server_playback:
+                    response.refresh()
+                f.response = response
+            elif self.options.replay_kill_extra:
+                ctx.log.warn(
+                    "server_playback: killed non-replay request {}".format(
+                        f.request.url
+                    )
+                )
+                f.response = http.HTTPResponse.make(404, b'', {'content-type': 'text/plain'})
+
+
+def start():
+    files = sys.argv[1:]
+    print("Replaying from files: {}".format(files))
+    return ServerPlayback(files)

testing/awsy/awsy/awsy_test_case.py

@@ -61,6 +61,9 @@ class AwsyTestCase(MarionetteTestCase):
         self.marionette.set_context('chrome')
         self._resultsDir = self.testvars["resultsDir"]
 
+        self._binary = self.testvars['bin']
+        self._run_local = self.testvars.get('run_local', False)
+
         # Cleanup our files from previous runs.
         for patt in ('memory-report-*.json.gz',
                      'perfherder_data.json',

testing/awsy/awsy/test_memory_usage.py

@@ -4,8 +4,12 @@
 
 import os
 import sys
+import yaml
+
+import mozinfo
 
 from marionette_driver.errors import JavascriptException, ScriptTimeoutException
+from mozproxy import get_playback
 
 AWSY_PATH = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 if AWSY_PATH not in sys.path:
@@ -36,12 +40,7 @@ class TestMemoryUsage(AwsyTestCase):
     def perf_checkpoints(self):
         return process_perf_data.CHECKPOINTS
 
-    def setUp(self):
-        AwsyTestCase.setUp(self)
-        self.logger.info("setting up")
-        self._webroot_dir = self.testvars["webRootDir"]
-        self._urls = []
-
+    def setupTp5(self):
         urls = None
         default_tp5n_manifest = os.path.join(self._webroot_dir, 'page_load_test', 'tp5n',
                                              'tp5n.manifest')
@@ -63,6 +62,84 @@ class TestMemoryUsage(AwsyTestCase):
         for url, server in zip(urls, self._webservers.servers):
             self._urls.append(url.strip().format(server.port))
 
+    def setupTp6(self):
+        # tp5n stores its manifest in the zip file that gets extracted, tp6
+        # doesn't so we just keep one in our project dir for now.
+        default_tp6_pages_manifest = os.path.join(AWSY_PATH, 'conf', 'tp6-pages.yml')
+        tp6_pages_manifest = self.testvars.get("pageManifest", default_tp6_pages_manifest)
+        urls = []
+        recordings = set()
+        with open(tp6_pages_manifest) as f:
+            d = yaml.safe_load(f)
+            for r in d:
+                recordings.add(r['rec'])
+                url = r['url']
+                if isinstance(url, list):
+                    urls.extend(url)
+                else:
+                    urls.append(url)
+
+        self._urls = urls
+
+        # Now we setup the mitm proxy with our tp6 pageset.
+        tp6_pageset_manifest = os.path.join(AWSY_PATH, 'tp6-pageset.manifest')
+        config = {
+            'playback_tool': 'mitmproxy',
+            'playback_binary_manifest': 'mitmproxy-rel-bin-{platform}.manifest',
+            'playback_pageset_manifest': tp6_pageset_manifest,
+            'platform': mozinfo.os,
+            'obj_path': self._webroot_dir,
+            'binary': self._binary,
+            'run_local': self._run_local,
+            'app': 'firefox',
+            'host': 'localhost',
+            'ignore_mitmdump_exit_failure': True,
+        }
+
+        self._playback = get_playback(config)
+
+        script = os.path.join(AWSY_PATH, "awsy", "alternate-server-replay.py")
+        recording_arg = []
+        for recording in recordings:
+            recording_arg.append(os.path.join(self._playback.mozproxy_dir, recording))
+
+        script = '""%s %s""' % (script, " ".join(recording_arg))
+
+        if mozinfo.os == "win":
+            script = script.replace("\\", "\\\\\\")
+
+        # --no-upstream-cert prevents mitmproxy from needing network access to
+        # the upstream servers
+        self._playback.config['playback_tool_args'] = [
+                "--no-upstream-cert",
+                "-s", script]
+
+        self.logger.info("Using script %s" % script)
+
+        self._playback.start()
+
+        # We need to reload after the mitmproxy cert is installed
+        self.marionette.restart(clean=False)
+
+        # Setup WebDriver capabilities that we need
+        self.marionette.delete_session()
+        caps = {
+                "unhandledPromptBehavior": "dismiss",  # Ignore page navigation warnings
+        }
+        self.marionette.start_session(caps)
+        self.marionette.set_context('chrome')
+
+    def setUp(self):
+        AwsyTestCase.setUp(self)
+        self.logger.info("setting up")
+        self._webroot_dir = self.testvars["webRootDir"]
+        self._urls = []
+
+        if self.testvars.get("tp6", False):
+            self.setupTp6()
+        else:
+            self.setupTp5()
+
         self.logger.info("areweslimyet run by %d pages, %d iterations,"
                          " %d perTabPause, %d settleWaitTime"
                          % (self._pages_to_load, self._iterations,
@@ -73,7 +150,11 @@ class TestMemoryUsage(AwsyTestCase):
         self.logger.info("tearing down!")
 
         self.logger.info("tearing down webservers!")
-        self._webservers.stop()
+
+        if self.testvars.get("tp6", False):
+            self._playback.stop()
+        else:
+            self._webservers.stop()
 
         AwsyTestCase.tearDown(self)
 

testing/awsy/conf/tp6-pages.yml

@@ -0,0 +1,52 @@
+- rec: wikia.mp
+  url: http://fandom.wikia.com/articles/fallout-76-will-live-and-die-on-the-creativity-of-its-playerbase
+- rec: google-docs.mp
+  url: https://docs.google.com/document/d/1US-07msg12slQtI_xchzYxcKlTs6Fp7WqIc6W5GK5M8/edit?usp=sharing
+- rec: google-slides.mp
+  url: https://docs.google.com/presentation/d/1Ici0ceWwpFvmIb3EmKeWSq_vAQdmmdFcWqaiLqUkJng/edit?usp=sharing
+- rec: google-sheets.mp
+  url: https://docs.google.com/spreadsheets/d/1jT9qfZFAeqNoOK97gruc34Zb7y_Q-O_drZ8kSXT-4D4/edit?usp=sharing
+- rec: wikipedia.mp
+  url: https://en.wikipedia.org/wiki/Barack_Obama
+- rec: imgur.mp
+  url: https://imgur.com/gallery/m5tYJL6
+- rec: google-mail.mp
+  url: https://mail.google.com/
+- rec: yahoo-mail.mp
+  url: https://mail.yahoo.com/
+- rec: pinterest.mp
+  url: https://pinterest.com/
+- rec: twitter.mp
+  url: https://twitter.com/BarackObama
+- rec: amazon.mp
+  url: https://www.amazon.com/s/url=search-alias%3Daps&field-keywords=laptop
+- rec: apple.mp
+  url: https://www.apple.com/macbook-pro/
+- rec: bing.mp
+  url: https://www.bing.com/search?q=barack+obama
+- rec: ebay.mp
+  url: https://www.ebay.com/
+- rec: facebook.mp
+  url: https://www.facebook.com
+- rec: google-search.mp
+  url:
+    - https://www.google.com/#hl=en&q=barack+obama
+    - https://www.google.com/search?hl=en&q=barack+obama&cad=h
+- rec: imdb.mp
+  url: https://www.imdb.com/title/tt0084967/?ref_=nv_sr_2
+- rec: instagram.mp
+  url: https://www.instagram.com/
+- rec: microsoft.mp
+  url: https://www.microsoft.com/en-us/windows/get-windows-10
+- rec: paypal.mp
+  url: https://www.paypal.com/myaccount/summary/
+- rec: reddit.mp
+  url: https://www.reddit.com/r/technology/comments/9sqwyh/we_posed_as_100_senators_to_run_ads_on_facebook/
+- rec: tumblr.mp
+  url: https://www.tumblr.com/dashboard
+- rec: yahoo-news.mp
+  url: https://www.yahoo.com/lifestyle/police-respond-noise-complaint-end-playing-video-games-respectful-tenants-002329963.html
+- rec: youtube.mp
+  url: https://www.youtube.com
+- rec: yandex.mp
+  url: https://yandex.ru/search/?text=barack%20obama&lr=10115

testing/awsy/conf/tp6-prefs.json

@@ -0,0 +1,15 @@
+{
+    "browser.newtabpage.enabled": true,
+    "browser.urlbar.userMadeSearchSuggestionsChoice": true,
+    "javascript.options.asyncstack": false,
+    "image.mem.surfacecache.min_expiration_ms": 10000,
+    "network.proxy.http": "localhost",
+    "network.proxy.http_port": 8080,
+    "network.proxy.ssl": "localhost",
+    "network.proxy.ssl_port": 8080,
+    "network.proxy.no_proxies_on": "localhost",
+    "network.proxy.type": 1,
+    "plugin.disable": true,
+    "startup.homepage_override_url": "",
+    "startup.homepage_welcome_url": ""
+}

testing/awsy/mach_commands.py

@@ -79,7 +79,7 @@ class MachCommands(MachCommandBase):
 
         runtime_testvars = {}
         for arg in ('webRootDir', 'pageManifest', 'resultsDir', 'entities', 'iterations',
-                    'perTabPause', 'settleWaitTime', 'maxTabs', 'dmd'):
+                    'perTabPause', 'settleWaitTime', 'maxTabs', 'dmd', 'tp6'):
             if arg in kwargs and kwargs[arg] is not None:
                 runtime_testvars[arg] = kwargs[arg]
 
@@ -94,6 +94,10 @@ class MachCommands(MachCommandBase):
         if 'resultsDir' not in runtime_testvars:
             runtime_testvars['resultsDir'] = os.path.join(awsy_tests_dir,
                                                           'results')
+
+        runtime_testvars['bin'] = binary
+        runtime_testvars['run_local'] = True
+
         page_load_test_dir = os.path.join(web_root_dir, 'page_load_test')
         if not os.path.isdir(page_load_test_dir):
             os.makedirs(page_load_test_dir)
@@ -238,6 +242,9 @@ class MachCommands(MachCommandBase):
     @CommandArgument('--dmd', group='AWSY', action='store_true',
                      dest='dmd', default=False,
                      help='Enable DMD during testing. Requires a DMD-enabled build.')
+    @CommandArgument('--tp6', group='AWSY', action='store_true',
+                     dest='tp6', default=False,
+                     help='Use the tp6 pageset during testing.')
     def run_awsy_test(self, tests, **kwargs):
         """mach awsy-test runs the in-tree version of the Are We Slim Yet
         (AWSY) tests.

testing/awsy/requirements.txt

@@ -1 +1,2 @@
 marionette-harness >= 4.0.0
+PyYaml >= 5.1

testing/awsy/setup.py

@@ -17,7 +17,7 @@ setup(
     license='MPL 1.1/GPL 2.0/LGPL 2.1',
     packages=find_packages(),
     zip_safe=False,
-    install_requires=["marionette_harness"],
+    install_requires=["marionette_harness", "PyYaml"],
     classifiers=['Development Status :: 4 - Beta',
                  'Environment :: Console',
                  'Intended Audience :: Developers',

testing/awsy/tp6-pageset.manifest

@@ -0,0 +1,132 @@
+[
+    {
+        "algorithm": "sha512",
+        "digest": "edc07e9eb78e3ec83364f5b9150a40c2640338957a7c53da848a1c0a5f0b917411a088d2ad7f2a5be0deb42dc5a2413534a1b682940f0e4c12f703d50b6ff1f2",
+        "filename": "mitmproxy-tp6-instagram-binast.zip",
+        "size": 6626956,
+        "unpack": true,
+        "visibility": "public"
+    },
+    {
+        "algorithm": "sha512",
+        "digest": "f1dba7631f1f17d04e4811a3ee139c23a8ff599bb1a38b564a05d87b02261f58168d3c3e8c49a1e16d3b268c9c315a0f7eb106eec1d759261641b729f691d013",
+        "filename": "mitmproxy-tp6-ebay.zip",
+        "size": 2860653,
+        "unpack": true,
+        "visibility": "public"
+    },
+    {
+        "algorithm": "sha512",
+        "digest": "28c3b2a5871bba34fbb25e1a9bbb367c665792484cffec40f2cb6b17d989569b9e56a51bf901ad195dcde45bc1dfa5c7428266147ffe5124e8dc8e1c2a7a9f3f",
+        "filename": "mitmproxy-tp6-google-mail.zip",
+        "size": 9966121,
+        "unpack": true,
+        "visibility": "public"
+    },
+    {
+        "algorithm": "sha512",
+        "digest": "913d7631cc2f1685310ea1d807cf4dc6a91298f6f24700393e46272128de9bb47ce44c9e88a771ff2169d0458c83b1df07b654456495b7d8fb6c6899fba0bc3a",
+        "filename": "mitmproxy-tp6-paypal.zip",
+        "size": 1037796,
+        "unpack": true,
+        "visibility": "public"
+    },
+    {
+        "algorithm": "sha512",
+        "digest": "a0dd4ccb99bac02a38b3c67baa08296c3e894837aba1c1b0750c592db6ff962ab0c61f557ffb45c2ea77e95345954a0786bf764448d090cc6cf76679d1a8a616",
+        "filename": "mitmproxy-tp6-pinterest.zip",
+        "size": 11627712,
+        "unpack": true,
+        "visibility": "public"
+    },
+    {
+        "algorithm": "sha512",
+        "digest": "3343f54b6c727a2061534872888da4e3aa647e81903c675dc318c717ed0c93f6ce4e2b98c66eb4128376cf8507590531283c95f3951259607edaaae28944d9a5",
+        "filename": "mitmproxy-recordings-raptor-tp6.zip",
+        "size": 8767174,
+        "unpack": true
+    },
+    {
+        "algorithm": "sha512",
+        "digest": "eb32218815d3c2187807f3a18d6df1657e01a215820deaaa7a035f06d180780ac9652479ac7db1d8cde3728a890521aff2e0a5c4738304966b2a93ce1b7f33bb",
+        "filename": "mitmproxy-recordings-raptor-gdocs.zip",
+        "size": 26179496,
+        "unpack": true
+    },
+    {
+        "algorithm": "sha512",
+        "digest": "5b8080cf842a50fe2055127daa084fa9f133307965e4d8af246978150b8b3b11f1fb06cdf65ba69e13c875b77db14e7494cdb940376b264c3aefb5b53e22b892",
+        "filename": "raptor-tp6-3.zip",
+        "size": 24512088,
+        "unpack": true
+    },
+    {
+        "algorithm": "sha512",
+        "digest": "0e9f2a23323f93f7a9839ab49d84555cce03bfab2196aa8670e604d6df390b22f0594527cecf7b2efd0449dd87507ec934bdcc74f1ebee68c22161f4104b6513",
+        "filename": "raptor-tp6-4.zip",
+        "size": 1643188,
+        "unpack": true
+    },
+    {
+        "algorithm": "sha512",
+        "digest": "4a52cb6770062231f9f283ab42ac53634d91677146b4d91931f508de22bff262512775d0c93d80730287000907e438d467136768883691c3e20c1e6a8f475a03",
+        "filename": "raptor-tp6-5.zip",
+        "size": 27670268,
+        "unpack": true
+    },
+    {
+        "algorithm": "sha512",
+        "digest": "21110ee297074413a72343bcb7d22ae88efce104de6e7eea5916e2dae004b29746ec6dd6155686cffed816e88d38ad1bbc04fe1253623ed8c0d4256a146d8b77",
+        "filename": "raptor-tp6-6.zip",
+        "size": 2240008,
+        "unpack": true
+    },
+    {
+        "algorithm": "sha512",
+        "digest": "3b5d06a79b2f9fb4e18534229844da69ca6f8e0f0759562947a29f681e27204ebae25b9a56c4ae2b95dabf067c3afeda424bcfb4d104bbc474a3970d4f2177aa",
+        "filename": "raptor-tp6-7.zip",
+        "size": 1862165,
+        "unpack": true,
+        "visibility": "public"
+    },
+    {
+        "algorithm": "sha512",
+        "digest": "d49ac687231a71a19d946e49ad9abea172316201d58ebd84bea360b70792948b36454966a0193039bafe18311444b3f25ab3f6101b77fa0a1e77b7ec83f3c8dc",
+        "filename": "mitmproxy-tp6-tumblr.zip",
+        "size": 20872384,
+        "unpack": true,
+        "visibility": "public"
+    },
+    {
+        "algorithm": "sha512",
+        "digest": "46ab2d3b58fe5be72029bae6ed2e84b8b81ac46d30a1ffaf4ac4ba3ba4284e73507d3becc2a274b9d5ef3497f91bb655fa48df959f0ed1c98e0581e638702200",
+        "filename": "mitmproxy-tp6-twitter.zip",
+        "size": 17821103,
+        "unpack": true,
+        "visibility": "public"
+    },
+    {
+        "algorithm": "sha512",
+        "digest": "efd7090b826d03f54ef58e631d8b24e0e7263e06fb679a8c2f072dde9e7a8223e02812db73e6fb71b08d4247f7652a72c94029959e34f6d49271c6172c0587cb",
+        "filename": "mitmproxy-tp6-wikipedia.zip",
+        "size": 1336699,
+        "unpack": true,
+        "visibility": "public"
+    },
+    {
+        "algorithm": "sha512",
+        "digest": "daf99106a71bea24ff6a0b195096fb36d328121effe9de84fc57e26a6ad8674db7c08eb17966cb2f3312a1e2b87b05da4339889c58b81c46c1e5af301dd27ad2",
+        "filename": "mitmproxy-tp6-yahoo-mail.zip",
+        "size": 4110884,
+        "unpack": true,
+        "visibility": "public"
+    },
+    {
+        "algorithm": "sha512",
+        "digest": "c29479b82a26db3722d6fe60c6a9d5b0849c828b4904553c115712610ead7be508d1c194050860140f8d9f4057b5e5088c085ea52319e37a0c9862fa120e5f22",
+        "filename": "mitmproxy-tp6-yahoo-news.zip",
+        "size": 11849146,
+        "unpack": true,
+        "visibility": "public"
+    }
+]