Use the new oslo py3kcompat.urlutils

Change-Id: I9e5d00a04460ef54d402b69f781d2a69a3bd0082

heat/api/openstack/v1/views/views_common.py

@@ -13,7 +13,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import urllib
+from heat.openstack.common.py3kcompat import urlutils
 
 
 def get_collection_links(request, items):
@@ -39,4 +39,4 @@ def _get_next_link(request, marker):
     params = request.params.copy()
     params['marker'] = marker
 
-    return "%s?%s" % (request.path_url, urllib.urlencode(params))
+    return "%s?%s" % (request.path_url, urlutils.urlencode(params))

heat/common/exception.py

@@ -18,10 +18,10 @@
 """Heat exception subclasses"""
 
 import functools
-import urlparse
 import sys
 
 from heat.openstack.common import log as logging
+from heat.openstack.common.py3kcompat import urlutils
 
 
 _FATAL_EXCEPTION_FORMAT_ERRORS = False
@@ -32,7 +32,7 @@ logger = logging.getLogger(__name__)
 
 class RedirectException(Exception):
     def __init__(self, url):
-        self.url = urlparse.urlparse(url)
+        self.url = urlutils.urlparse(url)
 
 
 class KeystoneError(Exception):

heat/common/identifier.py

@@ -13,11 +13,10 @@
 #    under the License.
 
 import re
-import urllib
-import urlparse
 import collections
 
 from heat.openstack.common import strutils
+from heat.openstack.common.py3kcompat import urlutils
 
 
 class HeatIdentifier(collections.Mapping):
@@ -63,10 +62,10 @@ class HeatIdentifier(collections.Mapping):
         if fields[1] != 'openstack' or fields[2] != 'heat' or not path:
             raise ValueError(_('"%s" is not a valid Heat ARN') % arn)
 
-        return cls(urllib.unquote(fields[4]),
+        return cls(urlutils.unquote(fields[4]),
-                   urllib.unquote(path.group(1)),
+                   urlutils.unquote(path.group(1)),
-                   urllib.unquote(path.group(2)),
+                   urlutils.unquote(path.group(2)),
-                   urllib.unquote(path.group(3)))
+                   urlutils.unquote(path.group(3)))
 
     @classmethod
     def from_arn_url(cls, url):
@@ -75,7 +74,7 @@ class HeatIdentifier(collections.Mapping):
         The URL is expected to contain a valid arn as part of the path
         '''
         # Sanity check the URL
-        urlp = urlparse.urlparse(url)
+        urlp = urlutils.urlparse(url)
         if (urlp.scheme not in ('http', 'https') or
                 not urlp.netloc or not urlp.path):
             raise ValueError(_('"%s" is not a valid URL') % url)
@@ -87,7 +86,7 @@ class HeatIdentifier(collections.Mapping):
             raise ValueError(_('"%s" is not a valid ARN URL') % url)
         # the +1 is to skip the leading /
         url_arn = urlp.path[match.start() + 1:]
-        arn = urllib.unquote(url_arn)
+        arn = urlutils.unquote(url_arn)
         return cls.from_arn(arn)
 
     def arn(self):
@@ -95,21 +94,21 @@ class HeatIdentifier(collections.Mapping):
         Return an ARN of the form:
             arn:openstack:heat::<tenant>:stacks/<stack_name>/<stack_id><path>
         '''
-        return 'arn:openstack:heat::%s:%s' % (urllib.quote(self.tenant, ''),
+        return 'arn:openstack:heat::%s:%s' % (urlutils.quote(self.tenant, ''),
                                               self._tenant_path())
 
     def arn_url_path(self):
         '''
         Return an ARN quoted correctly for use in a URL
         '''
-        return '/' + urllib.quote(self.arn(), '')
+        return '/' + urlutils.quote(self.arn(), '')
 
     def url_path(self):
         '''
         Return a URL-encoded path segment of a URL in the form:
             <tenant>/stacks/<stack_name>/<stack_id><path>
         '''
-        return '/'.join((urllib.quote(self.tenant, ''), self._tenant_path()))
+        return '/'.join((urlutils.quote(self.tenant, ''), self._tenant_path()))
 
     def _tenant_path(self):
         '''
@@ -117,9 +116,9 @@ class HeatIdentifier(collections.Mapping):
         in the form:
             stacks/<stack_name>/<stack_id><path>
         '''
-        return 'stacks/%s/%s%s' % (urllib.quote(self.stack_name, ''),
+        return 'stacks/%s/%s%s' % (urlutils.quote(self.stack_name, ''),
-                                   urllib.quote(self.stack_id, ''),
+                                   urlutils.quote(self.stack_id, ''),
-                                   urllib.quote(strutils.safe_encode(
+                                   urlutils.quote(strutils.safe_encode(
                                        self.path)))
 
     def _path_components(self):

heat/common/urlfetch.py

@@ -19,8 +19,6 @@ Utility for fetching a resource (e.g. a template) from a URL.
 
 import requests
 from requests import exceptions
-import urllib2
-import urlparse
 
 from oslo.config import cfg
 
@@ -28,6 +26,7 @@ cfg.CONF.import_opt('max_template_size', 'heat.common.config')
 
 from heat.openstack.common import log as logging
 from heat.openstack.common.gettextutils import _
+from heat.openstack.common.py3kcompat import urlutils
 
 logger = logging.getLogger(__name__)
 
@@ -43,15 +42,15 @@ def get(url, allowed_schemes=('http', 'https')):
     '''
     logger.info(_('Fetching data from %s') % url)
 
-    components = urlparse.urlparse(url)
+    components = urlutils.urlparse(url)
 
     if components.scheme not in allowed_schemes:
         raise IOError(_('Invalid URL scheme %s') % components.scheme)
 
     if components.scheme == 'file':
         try:
-            return urllib2.urlopen(url).read()
+            return urlutils.urlopen(url).read()
-        except urllib2.URLError as uex:
+        except urlutils.URLError as uex:
             raise IOError(_('Failed to retrieve template: %s') % str(uex))
 
     try:

heat/engine/resources/nova_utils.py

@@ -21,8 +21,6 @@ import json
 import os
 import pkgutil
 
-from urlparse import urlparse
-
 from oslo.config import cfg
 
 from heat.common import exception
@@ -31,6 +29,8 @@ from heat.engine import scheduler
 from heat.openstack.common import log as logging
 from heat.openstack.common.gettextutils import _
 from heat.openstack.common import uuidutils
+from heat.openstack.common.py3kcompat import urlutils
+
 
 logger = logging.getLogger(__name__)
 
@@ -175,8 +175,8 @@ def build_userdata(resource, userdata=None, instance_user=None):
 
     # Create a boto config which the cfntools on the host use to know
     # where the cfn and cw API's are to be accessed
-    cfn_url = urlparse(cfg.CONF.heat_metadata_server_url)
+    cfn_url = urlutils.urlparse(cfg.CONF.heat_metadata_server_url)
-    cw_url = urlparse(cfg.CONF.heat_watch_server_url)
+    cw_url = urlutils.urlparse(cfg.CONF.heat_watch_server_url)
     is_secure = cfg.CONF.instance_connection_is_secure
     vcerts = cfg.CONF.instance_connection_https_validate_certificates
     boto_cfg = "\n".join(["[Boto]",

heat/engine/resources/s3.py

@@ -13,11 +13,12 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-from urlparse import urlparse
-
 from heat.engine import clients
 from heat.engine import resource
+
 from heat.openstack.common import log as logging
+from heat.openstack.common.py3kcompat import urlutils
+
 
 logger = logging.getLogger(__name__)
 
@@ -126,7 +127,7 @@ class S3Bucket(resource.Resource):
 
     def _resolve_attribute(self, name):
         url = self.swift().get_auth()[0]
-        parsed = list(urlparse(url))
+        parsed = list(urlutils.urlparse(url))
         if name == 'DomainName':
             return parsed[1].split(':')[0]
         elif name == 'WebsiteURL':

heat/engine/resources/swift.py

@@ -13,13 +13,14 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-from urlparse import urlparse
-
 from heat.common import exception
 from heat.engine import resource
-from heat.openstack.common import log as logging
 from heat.engine import clients
 
+from heat.openstack.common import log as logging
+from heat.openstack.common.py3kcompat import urlutils
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -112,7 +113,7 @@ class SwiftContainer(resource.Resource):
 
     def FnGetAtt(self, key):
         url, token_id = self.swift().get_auth()
-        parsed = list(urlparse(url))
+        parsed = list(urlutils.urlparse(url))
         if key == 'DomainName':
             return parsed[1].split(':')[0]
         elif key == 'WebsiteURL':

heat/engine/signal_responder.py

@@ -13,9 +13,6 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import urllib
-import urlparse
-
 from oslo.config import cfg
 
 from keystoneclient.contrib.ec2 import utils as ec2_utils
@@ -27,6 +24,8 @@ from heat.engine import resource
 
 from heat.openstack.common import log
 from heat.openstack.common.gettextutils import _
+from heat.openstack.common.py3kcompat import urlutils
+
 
 LOG = log.getLogger(__name__)
 SIGNAL_TYPES = (
@@ -85,7 +84,7 @@ class SignalResponder(resource.Resource):
 
         waitcond_url = cfg.CONF.heat_waitcondition_server_url
         signal_url = waitcond_url.replace('/waitcondition', signal_type)
-        host_url = urlparse.urlparse(signal_url)
+        host_url = urlutils.urlparse(signal_url)
 
         path = self.identifier().arn_url_path()
         credentials = self.keystone().get_ec2_keypair(self.resource_id)
@@ -94,7 +93,7 @@ class SignalResponder(resource.Resource):
         # prcessing in the CFN API (ec2token.py) has an unquoted path, so we
         # need to calculate the signature with the path component unquoted, but
         # ensure the actual URL contains the quoted version...
-        unquoted_path = urllib.unquote(host_url.path + path)
+        unquoted_path = urlutils.unquote(host_url.path + path)
         request = {'host': host_url.netloc.lower(),
                    'verb': SIGNAL_VERB[signal_type],
                    'path': unquoted_path,
@@ -108,7 +107,7 @@ class SignalResponder(resource.Resource):
         signer = ec2_utils.Ec2Signer(credentials.secret)
         request['params']['Signature'] = signer.generate(request)
 
-        qs = urllib.urlencode(request['params'])
+        qs = urlutils.urlencode(request['params'])
         url = "%s%s?%s" % (signal_url.lower(),
                            path, qs)
 

heat/tests/test_api_openstack_v1_views_views_common.py

@@ -13,10 +13,10 @@
 #    under the License.
 
 import mock
-import urlparse
 
 from heat.tests.common import HeatTestCase
 from heat.api.openstack.v1.views import views_common
+from heat.openstack.common.py3kcompat import urlutils
 
 
 class TestViewsCommon(HeatTestCase):
@@ -74,10 +74,11 @@ class TestViewsCommon(HeatTestCase):
 
         next_link = filter(lambda link: link['rel'] == 'next', links).pop()
         url = next_link['href']
-        query_string = urlparse.urlparse(url).query
+        query_string = urlutils.urlparse(url).query
-        params = urlparse.parse_qs(query_string)
+        params = {}
-        self.assertEqual('2', params['limit'][0])
+        params.update(urlutils.parse_qsl(query_string))
-        self.assertEqual('bar', params['foo'][0])
+        self.assertEqual('2', params['limit'])
+        self.assertEqual('bar', params['foo'])
 
     def test_get_collection_links_handles_invalid_limits(self):
         self.setUpGetCollectionLinks()

heat/tests/test_urlfetch.py

@@ -15,7 +15,6 @@
 
 import requests
 from requests import exceptions
-import urllib2
 import cStringIO
 
 from oslo.config import cfg
@@ -23,6 +22,8 @@ from oslo.config import cfg
 from heat.common import urlfetch
 from heat.tests.common import HeatTestCase
 
+from heat.openstack.common.py3kcompat import urlutils
+
 
 class Response:
     def __init__(self, buf=''):
@@ -51,8 +52,8 @@ class UrlFetchTest(HeatTestCase):
         data = '{ "foo": "bar" }'
         url = 'file:///etc/profile'
 
-        self.m.StubOutWithMock(urllib2, 'urlopen')
+        self.m.StubOutWithMock(urlutils, 'urlopen')
-        urllib2.urlopen(url).AndReturn(cStringIO.StringIO(data))
+        urlutils.urlopen(url).AndReturn(cStringIO.StringIO(data))
         self.m.ReplayAll()
 
         self.assertEqual(data, urlfetch.get(url, allowed_schemes=['file']))
@@ -61,8 +62,8 @@ class UrlFetchTest(HeatTestCase):
     def test_file_scheme_failure(self):
         url = 'file:///etc/profile'
 
-        self.m.StubOutWithMock(urllib2, 'urlopen')
+        self.m.StubOutWithMock(urlutils, 'urlopen')
-        urllib2.urlopen(url).AndRaise(urllib2.URLError('oops'))
+        urlutils.urlopen(url).AndRaise(urlutils.URLError('oops'))
         self.m.ReplayAll()
 
         self.assertRaises(IOError, urlfetch.get, url, allowed_schemes=['file'])

heat/tests/v1_1/fakes.py

@@ -14,12 +14,13 @@
 # limitations under the License.
 
 import httplib2
-import urlparse
 
 from novaclient import client as base_client
 from novaclient.v1_1 import client
 from heat.tests import fakes
 
+from heat.openstack.common.py3kcompat import urlutils
+
 
 class FakeClient(fakes.FakeClient, client.Client):
 
@@ -45,7 +46,7 @@ class FakeHTTPClient(base_client.HTTPClient):
             assert 'body' in kwargs
 
         # Call the method
-        args = urlparse.parse_qsl(urlparse.urlparse(url)[4])
+        args = urlutils.parse_qsl(urlutils.urlparse(url)[4])
         kwargs.update(args)
         munged_url = url.rsplit('?', 1)[0]
         munged_url = munged_url.strip('/').replace('/', '_').replace('.', '_')