Bug 1521770 - Refactor telemetry-tests-client suite and remove firefox-puppeteer; r=janerik

Depends on D17648

Differential Revision: https://phabricator.services.mozilla.com/D17649

--HG--
rename : toolkit/components/telemetry/tests/marionette/harness/resources/helloworld/helloworld.html => toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/resources/helloworld/helloworld.html
rename : toolkit/components/telemetry/tests/marionette/harness/resources/helloworld/manifest.json => toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/resources/helloworld/manifest.json
extra : moz-landing-system : lando

toolkit/components/telemetry/tests/marionette/harness/MANIFEST.in

@@ -1,2 +1,3 @@
 exclude MANIFEST.in
-include requirements.txt
+include requirements.txt
+recursive-include telemetry_harness/resources *

toolkit/components/telemetry/tests/marionette/harness/requirements.txt

@@ -1,4 +1,2 @@
-firefox-puppeteer >= 52.1.0, <53.0.0
 marionette-harness >= 4.0.0
-requests==2.11.1
-simplejson
+requests==2.11.1

toolkit/components/telemetry/tests/marionette/harness/setup.py

@@ -6,7 +6,7 @@ import os
 
 from setuptools import setup, find_packages
 
-PACKAGE_VERSION = '0.1'
+PACKAGE_VERSION = "0.1"
 
 THIS_DIR = os.path.dirname(os.path.realpath(__name__))
 
@@ -16,29 +16,33 @@ def read(*parts):
         return f.read()
 
 
-setup(name='telemetry-harness',
-      version=PACKAGE_VERSION,
-      description=("""Custom Marionette runner classes and entry scripts for Telemetry
-specific Marionette tests."""),
-      classifiers=[
-          'Environment :: Console',
-          'Intended Audience :: Developers',
-          'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
-          'Natural Language :: English',
-          'Operating System :: OS Independent',
-          'Programming Language :: Python',
-          'Topic :: Software Development :: Libraries :: Python Modules',
-      ],
-      keywords='mozilla',
-      author='Firefox Test Engineering Team',
-      author_email='firefox-test-engineering@mozilla.org',
-      url='https://developer.mozilla.org/en-US/docs/Mozilla/QA/telemetry_harness',
-      license='MPL 2.0',
-      packages=find_packages(),
-      zip_safe=False,
-      install_requires=read('requirements.txt').splitlines(),
-      include_package_data=True,
-      entry_points="""
+setup(
+    name="telemetry-harness",
+    version=PACKAGE_VERSION,
+    description=(
+        "Custom Marionette runner classes and entry scripts for "
+        "Telemetry specific Marionette tests."
+    ),
+    classifiers=[
+        "Environment :: Console",
+        "Intended Audience :: Developers",
+        "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
+        "Natural Language :: English",
+        "Operating System :: OS Independent",
+        "Programming Language :: Python",
+        "Topic :: Software Development :: Libraries :: Python Modules",
+    ],
+    keywords="mozilla",
+    author="Firefox Test Engineering Team",
+    author_email="firefox-test-engineering@mozilla.org",
+    url="https://developer.mozilla.org/en-US/docs/Mozilla/QA/telemetry_harness",
+    license="MPL 2.0",
+    packages=find_packages(),
+    include_package_data=True,
+    zip_safe=False,
+    install_requires=read("requirements.txt").splitlines(),
+    entry_points="""
         [console_scripts]
         telemetry-harness = telemetry_harness.runtests:cli
-    """)
+    """,
+)

toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/ping_server.py

@@ -0,0 +1,69 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import zlib
+
+import mozlog
+
+from marionette_harness.runner import httpd
+
+
+class PingServer(object):
+    """HTTP server for receiving Firefox Client Telemetry pings."""
+
+    def __init__(self, server_root, url):
+        self._logger = mozlog.get_default_logger(component="pingserver")
+        self.pings = []
+
+        @httpd.handlers.handler
+        def pings_handler(request, response):
+            """Handler for HTTP requests to the ping server."""
+            request_data = request.body
+
+            if request.headers.get("Content-Encoding") == "gzip":
+                request_data = zlib.decompress(
+                    request_data, zlib.MAX_WBITS | 16
+                )
+
+            ping_data = json.loads(request_data)
+
+            # Store JSON data to self.pings to be used by wait_for_pings()
+            self.pings.append(ping_data)
+
+            ping_type = ping_data["type"]
+
+            log_message = "pings_handler received '{}' ping".format(ping_type)
+
+            if ping_type == "main":
+                ping_reason = ping_data["payload"]["info"]["reason"]
+                log_message = "{} with reason '{}'".format(
+                    log_message, ping_reason
+                )
+
+            self._logger.info(log_message)
+
+            status_code = 200
+            content = "OK"
+            headers = [
+                ("Content-Type", "text/plain"),
+                ("Content-Length", len(content)),
+            ]
+
+            return (status_code, headers, content)
+
+        self._httpd = httpd.FixtureServer(server_root, url=url)
+        self._httpd.router.register("POST", "/pings*", pings_handler)
+
+    def get_url(self, *args, **kwargs):
+        """Return a URL from the HTTP server."""
+        return self._httpd.get_url(*args, **kwargs)
+
+    def start(self):
+        """Start the HTTP server."""
+        return self._httpd.start()
+
+    def stop(self):
+        """Stop the HTTP server."""
+        return self._httpd.stop()

toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/runner.py

@@ -6,15 +6,41 @@
 from marionette_harness import BaseMarionetteTestRunner
 from testcase import TelemetryTestCase
 
+SERVER_URL = "http://127.0.0.1:8000"
+
 
 class TelemetryTestRunner(BaseMarionetteTestRunner):
+    """TestRunner for the telemetry-tests-client suite."""
 
     def __init__(self, **kwargs):
-        super(TelemetryTestRunner, self).__init__(**kwargs)
-        self.server_root = kwargs.pop('server_root')
-        self.testvars['server_root'] = self.server_root
+        """Set test variables and preferences specific to Firefox client
+        telemetry.
+        """
 
-        # select the appropriate GeckoInstance
-        self.app = 'fxdesktop'
+        # Select the appropriate GeckoInstance
+        kwargs["app"] = "fxdesktop"
+
+        prefs = kwargs.pop("prefs", {})
+
+        # Set Firefox Client Telemetry specific preferences
+        prefs.update(
+            {
+                "toolkit.telemetry.server": "{}/pings".format(SERVER_URL),
+                "toolkit.telemetry.initDelay": 1,
+                "toolkit.telemetry.minSubsessionLength": 0,
+                "datareporting.healthreport.uploadEnabled": True,
+                "datareporting.policy.dataSubmissionEnabled": True,
+                "datareporting.policy.dataSubmissionPolicyBypassNotification": True,
+                "toolkit.telemetry.log.level": "Trace",
+                "toolkit.telemetry.log.dump": True,
+                "toolkit.telemetry.send.overrideOfficialCheck": True,
+                "toolkit.telemetry.testing.disableFuzzingDelay": True,
+            }
+        )
+
+        super(TelemetryTestRunner, self).__init__(prefs=prefs, **kwargs)
+
+        self.testvars["server_root"] = kwargs["server_root"]
+        self.testvars["server_url"] = SERVER_URL
 
         self.test_handlers = [TelemetryTestCase]

toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/runtests.py

@@ -7,9 +7,8 @@ from runner import TelemetryTestRunner
 
 
 def cli(args=None):
-    mn_cli(runner_class=TelemetryTestRunner,
-           args=args)
+    mn_cli(runner_class=TelemetryTestRunner, args=args)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     cli()

toolkit/components/telemetry/tests/marionette/harness/telemetry_harness/testcase.py

@@ -2,18 +2,18 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import contextlib
 import os
 import re
-import simplejson as json
-import time
-import zlib
 
-from firefox_puppeteer import PuppeteerMixin
 from marionette_driver.addons import Addons
 from marionette_driver.errors import MarionetteException
 from marionette_driver.wait import Wait
+from marionette_driver import By, keys
 from marionette_harness import MarionetteTestCase
-from marionette_harness.runner import httpd
+from marionette_harness.runner.mixins.window_manager import WindowManagerMixin
+
+from ping_server import PingServer
 
 
 CANARY_CLIENT_ID = "c0ffeec0-ffee-c0ff-eec0-ffeec0ffeec0"
@@ -22,75 +22,56 @@ UUID_PATTERN = re.compile(
 )
 
 
-class TelemetryTestCase(PuppeteerMixin, MarionetteTestCase):
-
+class TelemetryTestCase(WindowManagerMixin, MarionetteTestCase):
     def __init__(self, *args, **kwargs):
+        """Initialize the test case and create a ping server."""
         super(TelemetryTestCase, self).__init__(*args, **kwargs)
-        self.pings = []
+
+        self.ping_server = PingServer(
+            self.testvars["server_root"], self.testvars["server_url"]
+        )
 
     def setUp(self, *args, **kwargs):
+        """Set up the test case and start the ping server."""
         super(TelemetryTestCase, self).setUp(*args, **kwargs)
 
-        @httpd.handlers.handler
-        def pings_handler(request, response):
-            """Handler for HTTP requests to the ping server."""
+        with self.marionette.using_context(self.marionette.CONTEXT_CONTENT):
+            self.marionette.navigate("about:about")
 
-            request_data = request.body
+        self.ping_server.start()
 
-            if request.headers.get("Content-Encoding") == "gzip":
-                request_data = zlib.decompress(request_data, zlib.MAX_WBITS | 16)
+    @contextlib.contextmanager
+    def new_tab(self):
+        """Perform operations in a new tab and then close the new tab."""
 
-            ping_data = json.loads(request_data)
+        with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
+            start_tab = self.marionette.current_window_handle
+            new_tab = self.open_tab(focus=True)
+            self.marionette.switch_to_window(new_tab)
 
-            # Store JSON data to self.pings to be used by wait_for_pings()
-            self.pings.append(ping_data)
+            yield
 
-            ping_type = ping_data["type"]
+            self.marionette.close()
+            self.marionette.switch_to_window(start_tab)
 
-            log_message = "pings_handler received '{}' ping".format(ping_type)
+    def search(self, text):
+        """Perform a search via the browser's URL bar."""
 
-            if ping_type == "main":
-                ping_reason = ping_data["payload"]["info"]["reason"]
-                log_message = "{} with reason '{}'".format(log_message, ping_reason)
+        with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
+            urlbar = self.marionette.find_element(By.ID, "urlbar")
+            urlbar.send_keys(keys.Keys.DELETE)
+            urlbar.send_keys(text + keys.Keys.ENTER)
 
-            self.logger.info(log_message)
+    def search_in_new_tab(self, text):
+        """Open a new tab and perform a search via the browser's URL bar,
+        then close the new tab."""
 
-            status_code = 200
-            content = "OK"
-            headers = [
-                ("Content-Type", "text/plain"),
-                ("Content-Length", len(content)),
-            ]
-
-            return (status_code, headers, content)
-
-        self.httpd = httpd.FixtureServer(self.testvars['server_root'])
-        self.httpd.router.register("POST", '/pings*', pings_handler)
-        self.httpd.start()
-
-        self.ping_server_url = '{}pings'.format(self.httpd.get_url('/'))
-
-        telemetry_prefs = {
-            'toolkit.telemetry.server': self.ping_server_url,
-            'toolkit.telemetry.initDelay': 1,
-            'toolkit.telemetry.minSubsessionLength': 0,
-            'datareporting.healthreport.uploadEnabled': True,
-            'datareporting.policy.dataSubmissionEnabled': True,
-            'datareporting.policy.dataSubmissionPolicyBypassNotification': True,
-            'toolkit.telemetry.log.level': 'Trace',
-            'toolkit.telemetry.log.dump': True,
-            'toolkit.telemetry.send.overrideOfficialCheck': True,
-            'toolkit.telemetry.testing.disableFuzzingDelay': True,
-        }
-
-        # Firefox will be forced to restart with the prefs enforced.
-        self.marionette.enforce_gecko_prefs(telemetry_prefs)
-
-        # Wait 5 seconds to ensure that telemetry has reinitialized
-        time.sleep(5)
+        with self.new_tab():
+            self.search(text)
 
     def assertIsValidUUID(self, value):
         """Check if the given UUID is valid."""
+
         self.assertIsNotNone(value)
         self.assertNotEqual(value, "")
 
@@ -109,14 +90,14 @@ class TelemetryTestCase(PuppeteerMixin, MarionetteTestCase):
         the `count` number of pings, that match the given filter.
         """
         # Keep track of the current number of pings
-        current_num_pings = len(self.pings)
+        current_num_pings = len(self.ping_server.pings)
 
         # New list to store new pings that satisfy the filter
         filtered_pings = []
 
         def wait_func(*args, **kwargs):
-            # Ignore existing pings in self.pings
-            new_pings = self.pings[current_num_pings:]
+            # Ignore existing pings in self.ping_server.pings
+            new_pings = self.ping_server.pings[current_num_pings:]
 
             # Filter pings to make sure we wait for the correct ping type
             filtered_pings[:] = [p for p in new_pings if ping_filter(p)]
@@ -125,7 +106,7 @@ class TelemetryTestCase(PuppeteerMixin, MarionetteTestCase):
 
         self.logger.info(
             "wait_for_pings running action '{action}'.".format(
-                action=action_func.__name__,
+                action=action_func.__name__
             )
         )
 
@@ -147,15 +128,13 @@ class TelemetryTestCase(PuppeteerMixin, MarionetteTestCase):
         return ping
 
     def restart_browser(self):
-        """Restarts browser while maintaining the same profile and session."""
-        self.restart(clean=False, in_app=True)
+        """Restarts browser while maintaining the same profile."""
+        return self.marionette.restart(clean=False, in_app=True)
 
     def install_addon(self):
         """Install a minimal addon."""
 
-        resources_dir = os.path.join(
-            os.path.dirname(os.path.dirname(__file__)), "resources"
-        )
+        resources_dir = os.path.join(os.path.dirname(__file__), "resources")
 
         addon_path = os.path.abspath(os.path.join(resources_dir, "helloworld"))
 
@@ -169,16 +148,25 @@ class TelemetryTestCase(PuppeteerMixin, MarionetteTestCase):
 
     @property
     def client_id(self):
-        return self.marionette.execute_script('Cu.import("resource://gre/modules/ClientID.jsm");'
-                                              'return ClientID.getCachedClientID();')
+        """Return the ID of the current client."""
+        with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
+            return self.marionette.execute_script(
+                'Cu.import("resource://gre/modules/ClientID.jsm");'
+                "return ClientID.getCachedClientID();"
+            )
 
     @property
     def subsession_id(self):
-        ping_data = self.marionette.execute_script(
-            'Cu.import("resource://gre/modules/TelemetryController.jsm");'
-            'return TelemetryController.getCurrentPingData(true);')
-        return ping_data[u'payload'][u'info'][u'subsessionId']
+        """Return the ID of the current subsession."""
+        with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
+            ping_data = self.marionette.execute_script(
+                'Cu.import("resource://gre/modules/TelemetryController.jsm");'
+                "return TelemetryController.getCurrentPingData(true);"
+            )
+            return ping_data[u"payload"][u"info"][u"subsessionId"]
 
     def tearDown(self, *args, **kwargs):
-        self.httpd.stop()
+        """Stop the ping server and tear down the testcase."""
         super(TelemetryTestCase, self).tearDown()
+        self.ping_server.stop()
+        self.marionette.quit(clean=True)

toolkit/components/telemetry/tests/marionette/tests/client/test_main_tab_scalars.py

@@ -13,18 +13,19 @@ class TestMainTabScalars(TelemetryTestCase):
         """Test for Telemetry Scalars."""
 
         with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
-            tab1 = self.browser.tabbar.selected_tab
+            start_tab = self.marionette.current_window_handle
 
-            tab2 = self.browser.tabbar.open_tab()
-            self.browser.tabbar.switch_to(tab2)
+            tab2 = self.open_tab(focus=True)
+            self.marionette.switch_to_window(tab2)
 
-            tab3 = self.browser.tabbar.open_tab()
-            self.browser.tabbar.switch_to(tab3)
+            tab3 = self.open_tab(focus=True)
+            self.marionette.switch_to_window(tab3)
 
-            self.browser.tabbar.close_tab(tab3, force=True)
-            self.browser.tabbar.close_tab(tab2, force=True)
+            self.marionette.close()
+            self.marionette.switch_to_window(tab2)
 
-            self.browser.tabbar.switch_to(tab1)
+            self.marionette.close()
+            self.marionette.switch_to_window(start_tab)
 
         ping = self.wait_for_ping(self.restart_browser, MAIN_SHUTDOWN_PING)
 
@@ -33,6 +34,10 @@ class TestMainTabScalars(TelemetryTestCase):
 
         scalars = ping["payload"]["processes"]["parent"]["scalars"]
 
-        self.assertEqual(scalars["browser.engagement.max_concurrent_tab_count"], 3)
+        self.assertEqual(
+            scalars["browser.engagement.max_concurrent_tab_count"], 3
+        )
         self.assertEqual(scalars["browser.engagement.tab_open_event_count"], 2)
-        self.assertEqual(scalars["browser.engagement.max_concurrent_window_count"], 1)
+        self.assertEqual(
+            scalars["browser.engagement.max_concurrent_window_count"], 1
+        )

toolkit/components/telemetry/tests/marionette/tests/client/test_search_counts_across_sessions.py

@@ -24,7 +24,8 @@ class TestSearchCounts(TelemetryTestCase):
         return searchService.defaultEngine.identifier;
         """
 
-        return self.marionette.execute_script(textwrap.dedent(script))
+        with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
+            return self.marionette.execute_script(textwrap.dedent(script))
 
     def setUp(self):
         """Set up the test case and store the identifier of the default
@@ -34,25 +35,6 @@ class TestSearchCounts(TelemetryTestCase):
         super(TestSearchCounts, self).setUp()
         self.search_engine = self.get_default_search_engine()
 
-    def search(self, text):
-        """Perform a search via the browser's location bar."""
-        with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
-            self.browser.navbar.locationbar.load_url(text)
-
-    def search_in_new_tab(self, text):
-        """Open a new tab and perform a search via the browser's location
-        bar, then close the new tab.
-        """
-        with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
-            new_tab = self.browser.tabbar.open_tab()
-            self.browser.tabbar.switch_to(new_tab)
-            self.browser.navbar.locationbar.load_url(text)
-            new_tab.close()
-
-    def restart_with_new_session(self):
-        """Restart the browser with the same profile."""
-        return self.restart(clean=False, in_app=True)
-
     def test_search_counts(self):
         """Test for SEARCH_COUNTS across sessions."""
 
@@ -64,9 +46,7 @@ class TestSearchCounts(TelemetryTestCase):
 
         self.search_in_new_tab("mozilla firefox")
 
-        ping1 = self.wait_for_ping(
-            self.restart_with_new_session, MAIN_SHUTDOWN_PING
-        )
+        ping1 = self.wait_for_ping(self.restart_browser, MAIN_SHUTDOWN_PING)
 
         # Session S2, subsession 1:
         # - Outcome 1
@@ -179,9 +159,7 @@ class TestSearchCounts(TelemetryTestCase):
         self.search("python unittest")
         self.search("python pytest")
 
-        ping3 = self.wait_for_ping(
-            self.restart_with_new_session, MAIN_SHUTDOWN_PING
-        )
+        ping3 = self.wait_for_ping(self.restart_browser, MAIN_SHUTDOWN_PING)
 
         # Session S3, subsession 1:
         # - Outcome 3

toolkit/components/telemetry/tests/marionette/tests/unit/test_ping_server_received_ping.py

@@ -8,6 +8,11 @@ from telemetry_harness.testcase import TelemetryTestCase
 
 
 class TestPingServer(TelemetryTestCase):
+    def setUp(self, *args, **kwargs):
+        """Set up the test case retrieve the pings URL."""
+        super(TestPingServer, self).setUp(*args, **kwargs)
+        self.pings_url = self.ping_server.get_url("/pings")
+
     def test_ping_server_received_ping(self):
         ping_type = "server-test-ping"
         ping_reason = "unit-test"
@@ -15,9 +20,14 @@ class TestPingServer(TelemetryTestCase):
         def send_ping_request():
             """Perform a POST request to the ping server."""
             data = {"type": ping_type, "reason": ping_reason}
-            headers = {"Content-type": "application/json", "Accept": "text/plain"}
+            headers = {
+                "Content-type": "application/json",
+                "Accept": "text/plain",
+            }
 
-            response = requests.post(self.ping_server_url, json=data, headers=headers)
+            response = requests.post(
+                self.pings_url, json=data, headers=headers
+            )
 
             self.assertEqual(
                 response.status_code,