Bug 1530350 - Update Raptor unit tests so disabling a subtest doesn't cause a unit test failure r=perftest-reviewers,Bebe

Differential Revision: https://phabricator.services.mozilla.com/D54904

--HG--
extra : moz-landing-system : lando

testing/raptor/raptor/raptor.ini

@@ -137,3 +137,7 @@
 # raptor scenario tests
 [include:tests/scenario/raptor-scn-power-idle.ini]
 [include:tests/scenario/raptor-scn-power-idle-bg.ini]
+
+# raptor unit tests
+# this is required for the manifest unit tests
+[include:tests/unittests/raptor-tp6-unittest.ini]

testing/raptor/raptor/tests/unittests/raptor-tp6-unittest.ini

@@ -0,0 +1,45 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6-unittest
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy
+page_cycles = 25
+page_timeout = 60000
+unit = ms
+lower_is_better = true
+gecko_profile_interval = 1
+gecko_profile_entries = 14000000
+alert_threshold = 2.0
+alert_on = fcp, loadtime
+
+[raptor-tp6-unittest-amazon-firefox]
+apps = firefox
+test_url = https://www.amazon.com/s?k=laptop&ref=nb_sb_noss_1
+playback_pageset_manifest = mitm4-linux-firefox-amazon.manifest
+playback_recordings = amazon.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6-unittest-facebook-firefox]
+apps = firefox
+test_url = https://www.facebook.com
+playback_pageset_manifest = mitm4-linux-firefox-facebook.manifest
+playback_recordings = facebook.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6-unittest-google-firefox]
+apps = firefox
+test_url = https://www.google.com/search?hl=en&q=barack+obama&cad=h
+playback_pageset_manifest = mitm4-linux-firefox-google-search.manifest
+playback_recordings = google-search.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6-unittest-youtube-firefox]
+apps = firefox
+test_url = https://www.youtube.com
+playback_pageset_manifest = mitm4-linux-firefox-youtube.manifest
+playback_recordings = youtube.mp
+measure = fnbpaint, fcp, dcf, loadtime

testing/raptor/test/conftest.py

@@ -82,7 +82,7 @@ def get_binary():
 @pytest.fixture
 def create_args():
     args = Namespace(app='firefox',
-                     test='raptor-tp6-1',
+                     test='raptor-tp6-unittest',
                      binary='path/to/binary',
                      gecko_profile=False,
                      debug_mode=False,

testing/raptor/test/test_manifest.py

@@ -166,8 +166,8 @@ def test_get_raptor_test_list_firefox(create_args):
     test_list = get_raptor_test_list(args, mozinfo.os)
     assert len(test_list) == 4
 
-    subtests = ['raptor-tp6-google-firefox', 'raptor-tp6-amazon-firefox',
-                'raptor-tp6-facebook-firefox', 'raptor-tp6-youtube-firefox']
+    subtests = ['raptor-tp6-unittest-google-firefox', 'raptor-tp6-unittest-amazon-firefox',
+                'raptor-tp6-unittest-facebook-firefox', 'raptor-tp6-unittest-youtube-firefox']
 
     for next_subtest in test_list:
         assert next_subtest['name'] in subtests