Replace pytest-responses with our own custom implementation. (#10410)

Fixes #10399

Closes #10378
Closes #10317

This also updates pytest to 4.1.1

conftest.py

@@ -38,6 +38,27 @@ def mock_elasticsearch():
     stop_es_mocks()
 
 
+@pytest.fixture(autouse=True)
+def start_responses_mocking(request):
+    """Enable ``responses`` this enforcing us to explicitly mark tests
+    that require internet usage.
+    """
+    marker = request.node.get_closest_marker('allow_external_http_requests')
+
+    if not marker:
+        responses.start()
+
+    yield
+
+    try:
+        if not marker:
+            responses.stop()
+            responses.reset()
+    except RuntimeError:
+        # responses patcher was already uninstalled
+        pass
+
+
 @pytest.fixture(autouse=True)
 def mock_basket(settings):
     """Mock Basket in tests by default.

requirements/tests.txt

@@ -25,9 +25,9 @@ py==1.7.0 \
     --hash=sha256:e76826342cefe3c3d5f7e8ee4316b80d1dd8a300781612ddbc765c17ba25a6c6 \
     --hash=sha256:bf92637198836372b520efcba9e020c330123be8ce527e535d185ed4b6f45694
 # pytest is required by amo-validator, pytest-base-url, pytest-cov, pytest-django, pytest-html, pytest-instafail, pytest-selenium, pytest-variables, pytest-xdist
-pytest==4.0.2 \
-    --hash=sha256:f812ea39a0153566be53d88f8de94839db1e8a05352ed8a49525d7d7f37861e9 \
-    --hash=sha256:f689bf2fc18c4585403348dd56f47d87780bf217c53ed9ae7a3e2d7faa45f8e9
+pytest==4.1.1 \
+    --hash=sha256:41568ea7ecb4a68d7f63837cf65b92ce8d0105e43196ff2b26622995bb3dc4b2 \
+    --hash=sha256:c3c573a29d7c9547fb90217ece8a8843aa0c1328a797e200290dc3d0b4b823be
 pytest-django==3.4.5 \
     --hash=sha256:e88e471d3d0f9acfb6293bb03d0ee8a33ed978734e92ea6b5312163a6c9e87cc \
     --hash=sha256:1a5d33be930e3172fa238643a380414dc369fe8fa4b3c3de25e59ed142950736
@@ -66,8 +66,6 @@ more-itertools==5.0.0 \
     --hash=sha256:38a936c0a6d98a38bcc2d03fdaaedaba9f412879461dd2ceff8d37564d6522e4
 pathtools==0.1.2 \
     --hash=sha256:7c35c5421a39bb82e58018febd90e3b6e5db34c5443aaaf742b3f33d4655f1c0
-pytest-responses==0.3.0 \
-    --hash=sha256:4556395e4d1d69620027e618c302f0dbe16eef05b1037dc5955c921b6d3bc0ee
 attrs==18.2.0 \
     --hash=sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb \
     --hash=sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69

setup.cfg

@@ -4,6 +4,7 @@ python_files=test*.py
 markers =
     es_tests: mark a test as an elasticsearch test.
     needs_locales_compilation: mark a test as needing compiled locales to work.
+    allow_external_http_requests: mark a test to allow external http requests and disable responses.
 norecursedirs =
     node_modules locale static media site-static user-media tmp
     templates fixtures migrations

src/olympia/amo/tests/test_monitor.py

@@ -66,7 +66,6 @@ class TestMonitor(TestCase):
         assert status == ''
         assert rabbitmq_results[0][1]
 
-    @responses.activate
     def test_signer(self):
         responses.add_passthru(settings.AUTOGRAPH_CONFIG['server_url'])
 

src/olympia/amo/tests/test_pytest_fixtures.py

@@ -1,4 +1,9 @@
 """Testing the pytest fixtures themselves which are declared in conftest.py."""
+import pytest
+import responses
+import requests
+
+from requests.exceptions import ConnectionError
 
 from olympia.access.models import Group
 
@@ -14,3 +19,18 @@ def test_mozilla_user(mozilla_user):
     admin_group = mozilla_user.groups.get()
     assert admin_group.name == 'Admins'
     assert admin_group.rules == '*:*'
+
+
+@pytest.mark.allow_external_http_requests
+def test_external_requests_enabled():
+    with pytest.raises(ConnectionError):
+        requests.get('http://example.invalid')
+
+    assert len(responses.calls) == 0
+
+
+def test_external_requests_disabled_by_default():
+    with pytest.raises(ConnectionError):
+        requests.get('http://example.invalid')
+
+    assert len(responses.calls) == 1

src/olympia/core/tests/test_certifi.py

@@ -7,7 +7,7 @@ from cryptography.hazmat.backends.openssl.backend import backend
 from olympia.amo.tests import TestCase
 
 
-@pytest.mark.withoutresponses
+@pytest.mark.allow_external_http_requests
 class TestCertifi(TestCase):
 
     def test_openssl_version(self):

src/olympia/devhub/tests/test_views.py

@@ -1262,7 +1262,6 @@ class TestUploadDetail(BaseUploadTest):
 
     @override_switch('akismet-spam-check', active=True)
     @override_switch('akismet-addon-action', active=False)
-    @responses.activate
     @override_settings(AKISMET_API_KEY=None)
     def test_akismet_reports_created_spam_outcome_logging_only(self):
         akismet_url = settings.AKISMET_API_URL.format(
@@ -1283,7 +1282,6 @@ class TestUploadDetail(BaseUploadTest):
 
     @override_switch('akismet-spam-check', active=True)
     @override_switch('akismet-addon-action', active=True)
-    @responses.activate
     @override_settings(AKISMET_API_KEY=None)
     def test_akismet_reports_created_spam_outcome_action_taken(self):
         akismet_url = settings.AKISMET_API_URL.format(

src/olympia/devhub/tests/test_views_submit.py

@@ -238,7 +238,6 @@ class TestAddonSubmitAgreementWithPostReviewEnabled(TestSubmitBase):
 
         assert 'recaptcha' in response.context['agreement_form'].errors
 
-    @responses.activate
     @override_switch('addon-submission-captcha', active=True)
     def test_read_dev_agreement_captcha_active_success(self):
         self.user.update(read_dev_agreement=None)

src/olympia/discovery/tests/test_commands.py

@@ -41,7 +41,6 @@ class TestExtractDiscoStringsCommand(TestCase):
             (settings.DISCOVERY_EDITORIAL_CONTENT_FILENAME, 'jinja2')
             in settings.PUENTE['DOMAIN_METHODS']['django'])
 
-    @responses.activate
     def test_basic(self):
         responses.add(
             responses.GET, settings.DISCOVERY_EDITORIAL_CONTENT_API,

src/olympia/files/tests/test_commands.py

@@ -179,7 +179,6 @@ class TestWebextUpdateDescriptions(TestCase):
         assert not Translation.objects.filter(
             locale='klingon').exists()
 
-    @responses.activate
     def test_add_descriptions(self):
         self._register_uris()
         assert WebextPermissionDescription.objects.count() == 0
@@ -198,7 +197,6 @@ class TestWebextUpdateDescriptions(TestCase):
         assert WebextPermissionDescription.objects.filter(
             name='oldpermission').exists()
 
-    @responses.activate
     def test_clear_then_add_descriptions(self):
         self._register_uris()
         # Add an existing permission that won't be updated and will be cleared.

src/olympia/lib/akismet/tests.py

@@ -48,7 +48,6 @@ class BaseAkismetReportsModelTest(object):
         # check still one call.
         comment_check_mock.assert_called_once()
 
-    @responses.activate
     @override_settings(AKISMET_API_KEY=None)
     def test_comment_check(self):
         report = self._create_report()
@@ -89,7 +88,6 @@ class BaseAkismetReportsModelTest(object):
             assert result == report.result == AkismetReport.UNKNOWN
             incr_mock.assert_called_with(statsd_str + '.fail')
 
-    @responses.activate
     @override_settings(AKISMET_API_KEY=None)
     def test_submit_spam(self):
         report = self._create_report()
@@ -119,7 +117,6 @@ class BaseAkismetReportsModelTest(object):
         report.update(reported=False)
         assert not report.submit_spam()
 
-    @responses.activate
     @override_settings(AKISMET_API_KEY=None)
     def test_submit_ham(self):
         report = self._create_report()

src/olympia/signing/tests/test_views.py

@@ -482,7 +482,6 @@ class TestUploadVersion(BaseUploadVersionTestMixin, TestCase):
 
     @override_switch('akismet-spam-check', active=True)
     @override_switch('akismet-addon-action', active=False)
-    @responses.activate
     @override_settings(AKISMET_API_KEY=None)
     def test_akismet_reports_created_spam_outcome_logging_only(self):
         akismet_url = settings.AKISMET_API_URL.format(
@@ -506,7 +505,6 @@ class TestUploadVersion(BaseUploadVersionTestMixin, TestCase):
 
     @override_switch('akismet-spam-check', active=True)
     @override_switch('akismet-addon-action', active=True)
-    @responses.activate
     @override_settings(AKISMET_API_KEY=None)
     def test_akismet_reports_created_spam_outcome_action_taken(self):
         akismet_url = settings.AKISMET_API_URL.format(

tests/ui/conftest.py

@@ -59,7 +59,8 @@ def selenium(selenium, request):
 
     """
     # Skip mobile test with marker 'desktop_only'
-    if request.node.get_marker('desktop_only') and request.param == MOBILE:
+    marker = request.node.get_closest_marker('desktop_only')
+    if marker and request.param == MOBILE:
         pytest.skip('Skipping mobile test')
     selenium.set_window_size(*request.param)
     return selenium
@@ -76,7 +77,7 @@ def fxa_account(request):
         fxa_account.email = os.environ['FXA_EMAIL']
         fxa_account.password = os.environ['FXA_PASSWORD']
     except KeyError:
-        if request.node.get_marker('fxa_login'):
+        if request.node.get_closest_marker('fxa_login'):
             pytest.skip(
                 'Skipping test because no fxa account was found.'
                 ' Are FXA_EMAIL and FXA_PASSWORD environment variables set?')

tests/ui/test_devhub.py

@@ -7,7 +7,7 @@ from pages.desktop.devhub import DevHub
 @pytest.mark.fxa_login
 @pytest.mark.desktop_only
 @pytest.mark.nondestructive
-@pytest.mark.withoutresponses
+@pytest.mark.allow_external_http_requests
 def test_devhub_home_loads_addons(base_url, selenium, devhub_login):
     """Test devhub home loads correct number of addons listed."""
     devhub = devhub_login
@@ -19,7 +19,7 @@ def test_devhub_home_loads_addons(base_url, selenium, devhub_login):
 @pytest.mark.fxa_login
 @pytest.mark.desktop_only
 @pytest.mark.nondestructive
-@pytest.mark.withoutresponses
+@pytest.mark.allow_external_http_requests
 def test_devhub_addon_edit_link_works(base_url, selenium, devhub_login):
     """Test addon edit link returns edit page."""
     devhub = devhub_login
@@ -31,7 +31,7 @@ def test_devhub_addon_edit_link_works(base_url, selenium, devhub_login):
 @pytest.mark.fxa_login
 @pytest.mark.desktop_only
 @pytest.mark.nondestructive
-@pytest.mark.withoutresponses
+@pytest.mark.allow_external_http_requests
 def test_devhub_addon_upload(base_url, selenium, devhub_upload):
     """Test uploading an addon via devhub."""
     'ui-test-addon-2' in devhub_upload.addons[-1].name
@@ -40,7 +40,7 @@ def test_devhub_addon_upload(base_url, selenium, devhub_upload):
 @pytest.mark.fxa_login
 @pytest.mark.desktop_only
 @pytest.mark.nondestructive
-@pytest.mark.withoutresponses
+@pytest.mark.allow_external_http_requests
 def test_devhub_logout(base_url, selenium, devhub_login):
     """Logging out from devhub."""
     assert devhub_login.logged_in

tests/ui/test_login.py

@@ -5,7 +5,7 @@ from pages.desktop.home import Home
 
 @pytest.mark.fxa_login
 @pytest.mark.skip("Addons frontend login not working")
-@pytest.mark.withoutresponses
+@pytest.mark.allow_external_http_requests
 def test_login(base_url, selenium, fxa_account):
     """User can login"""
     page = Home(selenium, base_url).open()
@@ -16,7 +16,7 @@ def test_login(base_url, selenium, fxa_account):
 
 @pytest.mark.skip(
     reason='https://bugzilla.mozilla.org/show_bug.cgi?id=1453779')
-@pytest.mark.withoutresponses
+@pytest.mark.allow_external_http_requests
 def test_logout(base_url, selenium, user):
     """User can logout"""
     page = Home(selenium, base_url).open()