Use from six.moves.urllib.parse instead of urlparse

In Python 3, urlparse module is gone.
To support both Python 2 and Python 3, "six.moves.urllib.parse"
should be used instead.

Change-Id: I0adddbdea5a80606907a1b78ca1a04d15f61452b
Closes-Bug: #1403433

rally/deploy/engines/multihost.py

@@ -13,9 +13,9 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import urlparse
 
 import six
+from six.moves.urllib import parse
 
 import rally
 from rally import consts
@@ -87,7 +87,7 @@ class MultihostEngine(engine.EngineFactory):
         self.controller, self.endpoints = self._deploy_node(
             self.config['controller'])
         endpoint = self.endpoints[0]
-        self.controller_ip = urlparse.urlparse(endpoint.auth_url).hostname
+        self.controller_ip = parse.urlparse(endpoint.auth_url).hostname
 
         for node_config in self.config['nodes']:
             self._update_controller_ip(node_config)

rally/osclients.py

@@ -13,8 +13,6 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import urlparse
-
 from ceilometerclient import client as ceilometer
 from cinderclient import client as cinder
 from designateclient import v1 as designate
@@ -29,6 +27,7 @@ from neutronclient.neutron import client as neutron
 from novaclient import client as nova
 from oslo.config import cfg
 from saharaclient import client as sahara
+from six.moves.urllib import parse
 from troveclient import client as trove
 from zaqarclient.queues import client as zaqar
 
@@ -100,7 +99,7 @@ class Clients(object):
         }
         kw = dict(self.endpoint.to_dict().items() + new_kw.items())
         if kw["endpoint_type"] == consts.EndpointType.PUBLIC:
-            mgmt_url = urlparse.urlparse(kw["auth_url"])
+            mgmt_url = parse.urlparse(kw["auth_url"])
             if (mgmt_url.port != kw["admin_port"] and
                     mgmt_url.scheme != "https"):
                 kw["endpoint"] = "{0}://{1}:{2}{3}".format(

rally/verification/verifiers/tempest/config.py

@@ -17,11 +17,11 @@ import datetime
 import inspect
 import os
 import time
-import urlparse
 
 from oslo.config import cfg
 import requests
 from six.moves import configparser
+from six.moves.urllib import parse
 
 from rally.common.i18n import _
 from rally import db
@@ -231,7 +231,7 @@ class TempestConf(object):
             self.conf.set(section_name, service,
                           str(service in self.available_services))
         horizon_url = ('http://' +
-                       urlparse.urlparse(self.endpoint['auth_url']).hostname)
+                       parse.urlparse(self.endpoint['auth_url']).hostname)
         horizon_availability = (requests.get(horizon_url).status_code == 200)
         # convert boolean to string because ConfigParser fails
         # on attempt to get option with boolean value

tests/hacking/README.rst

@@ -25,4 +25,5 @@ Rally Specific Commandments
  * [N330] - Ensure that ``dict.iteritems()`` is not used
  * [N331] - Ensure that ``basestring`` is not used
  * [N332] - Ensure that ``StringIO.StringIO`` is not used
-* [N340] - Ensure that we are importing always ``from rally import objects``
+ * [N333] - Ensure that ``urlparse`` is not used
+* [N340] - Ensure that we are importing always ``from rally import objects``

tests/hacking/checks.py

@@ -48,6 +48,7 @@ re_assert_equal_in_start_with_true_or_false = re.compile(
 re_iteritems_method = re.compile(r"\.iteritems\(\)")
 re_basestring_method = re.compile(r"(^|[\s,(\[=])basestring([\s,)\]]|$)")
 re_StringIO_method = re.compile(r"StringIO\.StringIO\(")
+re_urlparse_method = re.compile(r"(^|[\s=])urlparse\.")
 
 
 def _parse_assert_mock_str(line):
@@ -267,6 +268,18 @@ def check_StringIO_method(logical_line):
                   "rather than StringIO.StringIO.")
 
 
+def check_urlparse_method(logical_line):
+    """Check if urlparse is properly called for compatibility with Python 3
+
+    The correct form is six.moves.urllib.parse instead of "urlparse".
+
+    N333
+    """
+    res = re_urlparse_method.search(logical_line)
+    if res:
+        yield (0, "N333: Use six.moves.urllib.parse rather than urlparse.")
+
+
 def check_no_direct_rally_objects_import(logical_line, filename):
     """Check if rally.objects are properly imported.
 
@@ -297,4 +310,5 @@ def factory(register):
     register(check_iteritems_method)
     register(check_basestring_method)
     register(check_StringIO_method)
+    register(check_urlparse_method)
     register(check_no_direct_rally_objects_import)

tests/unit/aas/rest/base.py

@@ -19,7 +19,7 @@
 
 import pecan
 import pecan.testing
-from requests import utils
+from six.moves.urllib import parse
 
 from tests.unit import test
 
@@ -208,14 +208,14 @@ class PecanControllerTest(test.TestCase):
     def validate_link(self, link, bookmark=False):
         """Check if the given link can get correct data."""
         # removes the scheme and net location parts of the link
-        url_parts = list(utils.urlparse.urlparse(link))
+        url_parts = list(parse.urlparse(link))
         url_parts[0] = url_parts[1] = ''
 
         # bookmark link should not have the version in the URL
         if bookmark and url_parts[2].startswith(PATH_PREFIX):
             return False
 
-        full_path = utils.urlparse.urlunparse(url_parts)
+        full_path = parse.urlunparse(url_parts)
         try:
             self.get_json(full_path, path_prefix='')
             return True

tests/unit/test_hacking.py

@@ -139,9 +139,16 @@ class HackingTestCase(test.TestCase):
         self.assertEqual(len(list(checks.check_StringIO_method(
             "StringIO.StringIO()"))), 1)
 
-        self.assertEqual(len(list(checks.check_basestring_method(
+        self.assertEqual(len(list(checks.check_StringIO_method(
             "six.moves.StringIO()"))), 0)
 
+    def test_check_urlparse_method(self):
+        self.assertEqual(len(list(checks.check_urlparse_method(
+            "urlparse.urlparse(url)"))), 1)
+
+        self.assertEqual(len(list(checks.check_urlparse_method(
+            "six.moves.urllib.parse.urlparse(url)"))), 0)
+
     def test_assert_equal_none(self):
         self.assertEqual(len(list(checks.assert_equal_none(
             "self.assertEqual(A, None)"))), 1)

tests/unit/test_osclients.py

@@ -13,11 +13,11 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import urlparse
 
 from keystoneclient import exceptions as keystone_exceptions
 import mock
 from oslo.config import cfg
+from six.moves.urllib import parse
 
 from rally import consts
 from rally import exceptions
@@ -55,7 +55,7 @@ class OSClientsTestCase(test.TestCase):
         self.assertNotIn("keystone", self.clients.cache)
         client = self.clients.keystone()
         self.assertEqual(client, self.fake_keystone)
-        mgmt_url = urlparse.urlparse(self.endpoint.auth_url)
+        mgmt_url = parse.urlparse(self.endpoint.auth_url)
         auth_url = "{0}://{1}:{2}{3}".format(mgmt_url.scheme,
                                              mgmt_url.hostname,
                                              self.endpoint.admin_port,