Replace urlparse imports by six.moves.urllib_parse for Python 3 compatibility

This keeps backwards-compatibility to Python 2 but moves us closer to running
with Python 3.

settings.py

@@ -5,7 +5,7 @@ won't be tracked in git).
 
 """
 import os
-from urlparse import urlparse
+from six.moves.urllib_parse import urlparse
 
 from olympia.lib.settings_base import *  # noqa
 

src/olympia/accounts/tests/test_utils.py

@@ -1,17 +1,16 @@
 # -*- coding: utf-8 -*-
 import json
+import mock
 import time
-import urlparse
 
 from base64 import urlsafe_b64decode, urlsafe_b64encode
 from datetime import datetime
+from six.moves.urllib_parse import parse_qs, urlparse
 
 from django.contrib.auth.models import AnonymousUser
 from django.test import RequestFactory
 from django.test.utils import override_settings
 
-import mock
-
 from olympia.accounts import utils
 from olympia.accounts.utils import process_fxa_event
 from olympia.amo.tests import TestCase, user_factory
@@ -64,11 +63,11 @@ def test_default_fxa_login_url_with_state():
     request = RequestFactory().get(path)
     request.session = {'fxa_state': 'myfxastate'}
     raw_url = utils.default_fxa_login_url(request)
-    url = urlparse.urlparse(raw_url)
+    url = urlparse(raw_url)
     base = '{scheme}://{netloc}{path}'.format(
         scheme=url.scheme, netloc=url.netloc, path=url.path)
     assert base == 'https://accounts.firefox.com/oauth/authorization'
-    query = urlparse.parse_qs(url.query)
+    query = parse_qs(url.query)
     next_path = urlsafe_b64encode(path).rstrip('=')
     assert query == {
         'action': ['signin'],
@@ -85,11 +84,11 @@ def test_default_fxa_register_url_with_state():
     request = RequestFactory().get(path)
     request.session = {'fxa_state': 'myfxastate'}
     raw_url = utils.default_fxa_register_url(request)
-    url = urlparse.urlparse(raw_url)
+    url = urlparse(raw_url)
     base = '{scheme}://{netloc}{path}'.format(
         scheme=url.scheme, netloc=url.netloc, path=url.path)
     assert base == 'https://accounts.firefox.com/oauth/authorization'
-    query = urlparse.parse_qs(url.query)
+    query = parse_qs(url.query)
     next_path = urlsafe_b64encode(path).rstrip('=')
     assert query == {
         'action': ['signup'],
@@ -111,11 +110,11 @@ def test_fxa_login_url_without_requiring_two_factor_auth():
         state=request.session['fxa_state'], next_path=path, action='signin',
         force_two_factor=False)
 
-    url = urlparse.urlparse(raw_url)
+    url = urlparse(raw_url)
     base = '{scheme}://{netloc}{path}'.format(
         scheme=url.scheme, netloc=url.netloc, path=url.path)
     assert base == 'https://accounts.firefox.com/oauth/authorization'
-    query = urlparse.parse_qs(url.query)
+    query = parse_qs(url.query)
     next_path = urlsafe_b64encode(path).rstrip('=')
     assert query == {
         'action': ['signin'],
@@ -137,11 +136,11 @@ def test_fxa_login_url_requiring_two_factor_auth():
         state=request.session['fxa_state'], next_path=path, action='signin',
         force_two_factor=True)
 
-    url = urlparse.urlparse(raw_url)
+    url = urlparse(raw_url)
     base = '{scheme}://{netloc}{path}'.format(
         scheme=url.scheme, netloc=url.netloc, path=url.path)
     assert base == 'https://accounts.firefox.com/oauth/authorization'
-    query = urlparse.parse_qs(url.query)
+    query = parse_qs(url.query)
     next_path = urlsafe_b64encode(path).rstrip('=')
     assert query == {
         'acr_values': ['AAL2'],
@@ -158,7 +157,7 @@ def test_unicode_next_path():
     request = RequestFactory().get(path)
     request.session = {}
     url = utils.default_fxa_login_url(request)
-    state = urlparse.parse_qs(urlparse.urlparse(url).query)['state'][0]
+    state = parse_qs(urlparse(url).query)['state'][0]
     next_path = urlsafe_b64decode(state.split(':')[1] + '===')
     assert next_path.decode('utf-8') == path
 

src/olympia/accounts/tests/test_views.py

@@ -1,9 +1,10 @@
 # -*- coding: utf-8 -*-
 import base64
 import json
-import urlparse
+import mock
 
 from os import path
+from six.moves.urllib_parse import parse_qs, urlparse
 
 from django.conf import settings
 from django.contrib.auth.models import AnonymousUser
@@ -12,8 +13,6 @@ from django.urls import reverse
 from django.test import RequestFactory
 from django.test.utils import override_settings
 
-import mock
-
 from rest_framework.exceptions import PermissionDenied
 from rest_framework.settings import api_settings
 from rest_framework.test import APIClient, APIRequestFactory
@@ -93,11 +92,11 @@ class TestLoginStartBaseView(WithDynamicEndpoints, TestCase):
                         lambda: 'arandomstring'):
             response = self.client.get(self.url)
         assert response.status_code == 302
-        url = urlparse.urlparse(response['location'])
+        url = urlparse(response['location'])
         redirect = '{scheme}://{netloc}{path}'.format(
             scheme=url.scheme, netloc=url.netloc, path=url.path)
         assert redirect == 'https://accounts.firefox.com/v1/authorization'
-        assert urlparse.parse_qs(url.query) == {
+        assert parse_qs(url.query) == {
             'action': ['signin'],
             'client_id': ['amodefault'],
             'redirect_url': ['https://addons.mozilla.org/fxa-authenticate'],
@@ -120,8 +119,8 @@ class TestLoginStartBaseView(WithDynamicEndpoints, TestCase):
                         lambda: state):
             response = self.client.get(self.url, data={'to': path})
         assert self.client.session['fxa_state'] == state
-        url = urlparse.urlparse(response['location'])
-        query = urlparse.parse_qs(url.query)
+        url = urlparse(response['location'])
+        query = parse_qs(url.query)
         state_parts = query['state'][0].split(':')
         assert len(state_parts) == 2
         assert state_parts[0] == state
@@ -133,8 +132,8 @@ class TestLoginStartBaseView(WithDynamicEndpoints, TestCase):
         self.initialize_session({})
         response = self.client.get(
             '{url}?to={path}'.format(path=path, url=self.url))
-        url = urlparse.urlparse(response['location'])
-        query = urlparse.parse_qs(url.query)
+        url = urlparse(response['location'])
+        query = parse_qs(url.query)
         assert ':' not in query['state'][0]
 
 
@@ -599,14 +598,14 @@ class TestWithUser(TestCase):
         # Query params should be kept on the redirect to FxA, with
         # acr_values=AAL2 added to force two-factor auth on FxA side.
         assert response.status_code == 302
-        url = urlparse.urlparse(response['Location'])
+        url = urlparse(response['Location'])
         base = '{scheme}://{netloc}{path}'.format(
             scheme=url.scheme, netloc=url.netloc, path=url.path)
         fxa_config = settings.FXA_CONFIG[settings.DEFAULT_FXA_CONFIG_NAME]
         assert base == '{host}{path}'.format(
             host=fxa_config['oauth_host'],
             path='/authorization')
-        query = urlparse.parse_qs(url.query)
+        query = parse_qs(url.query)
         next_path = base64.urlsafe_b64encode('/a/path/?').rstrip('=')
         assert query == {
             'acr_values': ['AAL2'],

src/olympia/addons/forms.py

@@ -1,7 +1,7 @@
 import os
 
 from datetime import datetime
-from urlparse import urlsplit
+from six.moves.urllib_parse import urlsplit
 
 from django import forms
 from django.conf import settings

src/olympia/addons/models.py

@@ -6,11 +6,11 @@ import os
 import posixpath
 import re
 import time
-import urlparse
 import uuid
 
 from datetime import datetime
 from operator import attrgetter
+from six.moves.urllib_parse import urlsplit
 
 from django.conf import settings
 from django.core.exceptions import ObjectDoesNotExist
@@ -2172,7 +2172,7 @@ class ReplacementAddon(ModelBase):
 
     @staticmethod
     def path_is_external(path):
-        return urlparse.urlsplit(path).scheme in ['http', 'https']
+        return urlsplit(path).scheme in ['http', 'https']
 
     def has_external_url(self):
         return self.path_is_external(self.path)

src/olympia/amo/templatetags/jinja_helpers.py

@@ -4,7 +4,7 @@ import os
 import random
 
 from operator import attrgetter
-from urlparse import urljoin
+from six.moves.urllib_parse import urljoin
 
 from django.conf import settings
 from django.core.exceptions import ObjectDoesNotExist

src/olympia/amo/tests/__init__.py

@@ -8,8 +8,9 @@ from contextlib import contextmanager
 from datetime import datetime, timedelta
 from functools import partial
 from importlib import import_module
+from six.moves.urllib_parse import parse_qs, urlparse
 from tempfile import NamedTemporaryFile
-from urlparse import parse_qs, urlparse
+
 
 from django import forms, test
 from django.conf import settings

src/olympia/amo/tests/test_helpers.py

@@ -3,7 +3,7 @@ import mimetypes
 import os
 
 from datetime import datetime, timedelta
-from urlparse import urljoin
+from six.moves.urllib_parse import urljoin
 
 from django.conf import settings
 from django.core.files.uploadedfile import SimpleUploadedFile

src/olympia/amo/urlresolvers.py

@@ -9,7 +9,7 @@ from threading import local
 from django import urls
 from django.conf import settings
 from django.utils.encoding import force_bytes
-from django.utils.http import _urlparse as urlparse
+from django.utils.http import _urlparse as django_urlparse
 from django.utils.translation.trans_real import parse_accept_lang_header
 
 import bleach
@@ -172,7 +172,9 @@ def get_outgoing_url(url):
     if not settings.REDIRECT_URL:
         return url
 
-    parsed_url = urlparse(url)
+    # django.utils.http._urlparse is a copy of python's urlparse()
+    # "but uses fixed urlsplit() function".
+    parsed_url = django_urlparse(url)
     url_netloc = parsed_url.netloc
 
     # This prevents a link like javascript://addons.mozilla.org...
@@ -182,7 +184,7 @@ def get_outgoing_url(url):
         return '/'
 
     # No double-escaping, and some domain names are excluded.
-    if (url_netloc == urlparse(settings.REDIRECT_URL).netloc or
+    if (url_netloc == django_urlparse(settings.REDIRECT_URL).netloc or
             url_netloc in settings.REDIRECT_URL_ALLOW_LIST):
         return url
 

src/olympia/amo/utils.py

@@ -13,11 +13,12 @@ import shutil
 import time
 import unicodedata
 import urllib
-import urlparse
 import string
 import subprocess
 import scandir
 
+from six.moves.urllib_parse import parse_qsl, ParseResult
+
 import django.core.mail
 
 from django.conf import settings
@@ -85,14 +86,14 @@ def urlparams(url_, hash=None, **query):
 
     # Use dict(parse_qsl) so we don't get lists of values.
     q = url.query
-    query_dict = dict(urlparse.parse_qsl(force_bytes(q))) if q else {}
+    query_dict = dict(parse_qsl(force_bytes(q))) if q else {}
     query_dict.update(
         (k, force_bytes(v) if v is not None else v) for k, v in query.items())
     query_string = urlencode(
         [(k, urllib.unquote(v)) for k, v in query_dict.items()
          if v is not None])
-    new = urlparse.ParseResult(url.scheme, url.netloc, url.path, url.params,
-                               query_string, fragment)
+    new = ParseResult(url.scheme, url.netloc, url.path, url.params,
+                      query_string, fragment)
     return new.geturl()
 
 

src/olympia/browse/tests.py

@@ -1,12 +1,12 @@
 # -*- coding: utf-8 -*-
 import re
 
-from urlparse import urlparse
+from six.moves.urllib_parse import urlparse
 
 from django.conf import settings
 from django.core.cache import cache
 from django.test.utils import override_settings
-from django.utils import http as urllib
+from django.utils.http import urlquote
 from django.utils.translation import trim_whitespace
 
 import mock
@@ -1018,8 +1018,7 @@ class TestCategoriesFeed(TestCase):
 
     def test_item_guid(self):
         t = self.feed.item_guid(self.addon)
-        url = u'/addon/%s/versions/v%s' % (self.addon.slug,
-                                           urllib.urlquote(self.u))
+        url = u'/addon/%s/versions/v%s' % (self.addon.slug, urlquote(self.u))
         assert t.endswith(url), t
 
 

src/olympia/discovery/utils.py

@@ -1,6 +1,6 @@
 import json
-import urlparse
 from collections import OrderedDict
+from six.moves.urllib_parse import urljoin
 
 from django.conf import settings
 from django.utils.http import urlencode
@@ -18,7 +18,7 @@ log = olympia.core.logger.getLogger('z.amo')
 
 def call_recommendation_server(id_or_guid, params, server):
     params = OrderedDict(sorted(params.items(), key=lambda t: t[0]))
-    endpoint = urlparse.urljoin(
+    endpoint = urljoin(
         server,
         '%s/%s%s' % (id_or_guid, '?' if params else '', urlencode(params)))
     log.debug(u'Calling recommendation server: {0}'.format(endpoint))

src/olympia/files/tests/test_views.py

@@ -2,7 +2,7 @@
 import json
 import os
 import shutil
-import urlparse
+from six.moves.urllib_parse import urlparse
 
 from django.conf import settings
 from django.core.cache import cache
@@ -484,7 +484,7 @@ class TestFileViewer(FilesBase, TestCase):
         res = self.client.get(self.files_redirect(binary))
         assert res.status_code == 302
         url = res['Location']
-        assert urlparse.urlparse(url).query.startswith('token=')
+        assert urlparse(url).query.startswith('token=')
 
     def test_memcache_goes_bye_bye(self):
         self.file_viewer.extract()
@@ -493,7 +493,7 @@ class TestFileViewer(FilesBase, TestCase):
         res = self.client.get(url)
         assert res.status_code == 200
 
-        token = urlparse.urlparse(url).query[6:]
+        token = urlparse(url).query[6:]
         cache.delete('token:{}'.format(token))
 
         res = self.client.get(url)

src/olympia/reviewers/tests/test_views.py

@@ -2,10 +2,10 @@
 import json
 import os
 import time
-import urlparse
 
 from collections import OrderedDict
 from datetime import datetime, timedelta
+from six.moves.urllib_parse import parse_qs
 
 from django.conf import settings
 from django.core import mail
@@ -1232,7 +1232,7 @@ class TestQueueBasics(QueueTest):
             # Update expected GET parameters with sort type.
             params.update(sort=[sort])
             # Parse querystring of link to make sure `sort` type is correct.
-            assert urlparse.parse_qs(a.attr('href').split('?')[1]) == params
+            assert parse_qs(a.attr('href').split('?')[1]) == params
 
     def test_no_results(self):
         response = self.client.get(self.url)

src/olympia/search/tests/test_views.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 import json
-import urlparse
+from six.moves.urllib_parse import parse_qs, parse_qsl, urlparse
 
 from django.http import QueryDict
 from django.test.client import RequestFactory
@@ -196,10 +196,10 @@ class TestESSearch(SearchBase):
         assert response.status_code == 302
         expected_params = 'q=f%C3%B4o&atype=1'
         redirected = response.url
-        parsed = urlparse.urlparse(redirected)
+        parsed = urlparse(redirected)
         params = parsed.query
         assert parsed.path == self.url
-        assert urlparse.parse_qs(params) == urlparse.parse_qs(expected_params)
+        assert parse_qs(params) == parse_qs(expected_params)
 
     def test_legacy_redirects(self):
         response = self.client.get(self.url + '?sort=averagerating')
@@ -212,23 +212,23 @@ class TestESSearch(SearchBase):
                  '&tag=dearbhair&cat=4%2C84')
         to = ('?sort=updated&advancedsearch=1&appver=1.0'
               '&tag=dearbhair&cat=4%2C84')
-        r = self.client.get(url + from_)
-        assert r.status_code == 301
-        redirected = r.url
-        parsed = urlparse.urlparse(redirected)
+        response = self.client.get(url + from_)
+        assert response.status_code == 301
+        redirected = response.url
+        parsed = urlparse(redirected)
         params = parsed.query
         assert parsed.path == url
-        assert urlparse.parse_qs(params) == urlparse.parse_qs(to[1:])
+        assert parse_qs(params) == parse_qs(to[1:])
 
     def check_platform_filters(self, platform, expected=None):
-        r = self.client.get('%s?platform=%s' % (self.url, platform),
-                            follow=True)
-        plats = r.context['platforms']
-        for idx, plat in enumerate(plats):
+        response = self.client.get('%s?platform=%s' % (self.url, platform),
+                                   follow=True)
+        platforms = response.context['platforms']
+        for idx, platform in enumerate(platforms):
             name, selected = expected[idx]
-            label = unicode(plat.text)
+            label = unicode(platform.text)
             assert label == name
-            assert plat.selected == selected
+            assert platform.selected == selected
 
     def test_platform_default(self):
         expected = [
@@ -868,7 +868,7 @@ class TestPersonaSearch(SearchBase):
 ])
 def test_search_redirects(test_input, expected):
     assert views.fix_search_query(QueryDict(test_input)) == (
-        dict(urlparse.parse_qsl(expected)))
+        dict(parse_qsl(expected)))
 
 
 @pytest.mark.parametrize("test_input", [

src/olympia/stats/management/commands/__init__.py

@@ -1,7 +1,7 @@
 import boto3
 import codecs
 from io import StringIO
-from urlparse import urlparse
+from six.moves.urllib_parse import urlparse
 
 from django.core.files.storage import get_storage_class
 

src/olympia/users/tests/test_helpers.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 import re
-import urlparse
+from six.moves.urllib_parse import parse_qs, urlparse
 
 import mock
 import pytest
@@ -128,11 +128,11 @@ class TestAddonUsersList(TestPersonas, TestCase):
 
 def test_manage_fxa_link():
     user = mock.MagicMock(email='me@someplace.ca', fxa_id='abcd1234')
-    link = urlparse.urlparse(manage_fxa_link({'user': user}))
+    link = urlparse(manage_fxa_link({'user': user}))
     url = '{scheme}://{netloc}{path}'.format(
         scheme=link.scheme, netloc=link.netloc, path=link.path)
     assert url == 'https://stable.dev.lcip.org/settings'
-    query = urlparse.parse_qs(link.query)
+    query = parse_qs(link.query)
     assert query == {
         'uid': ['abcd1234'],
         'email': ['me@someplace.ca'],

src/olympia/users/tests/test_views.py

@@ -1,7 +1,7 @@
 import json
 
 from datetime import datetime, timedelta
-from urlparse import urlparse
+from six.moves.urllib_parse import urlparse
 
 from django.conf import settings
 from django.core import mail