diff --git a/heat/api/openstack/v1/views/views_common.py b/heat/api/openstack/v1/views/views_common.py index 9bcdda14c5..67524ef9d8 100644 --- a/heat/api/openstack/v1/views/views_common.py +++ b/heat/api/openstack/v1/views/views_common.py @@ -13,7 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. -import urllib +from heat.openstack.common.py3kcompat import urlutils def get_collection_links(request, items): @@ -39,4 +39,4 @@ def _get_next_link(request, marker): params = request.params.copy() params['marker'] = marker - return "%s?%s" % (request.path_url, urllib.urlencode(params)) + return "%s?%s" % (request.path_url, urlutils.urlencode(params)) diff --git a/heat/common/exception.py b/heat/common/exception.py index 54d3e121e0..8fe68c6de6 100644 --- a/heat/common/exception.py +++ b/heat/common/exception.py @@ -18,10 +18,10 @@ """Heat exception subclasses""" import functools -import urlparse import sys from heat.openstack.common import log as logging +from heat.openstack.common.py3kcompat import urlutils _FATAL_EXCEPTION_FORMAT_ERRORS = False @@ -32,7 +32,7 @@ logger = logging.getLogger(__name__) class RedirectException(Exception): def __init__(self, url): - self.url = urlparse.urlparse(url) + self.url = urlutils.urlparse(url) class KeystoneError(Exception): diff --git a/heat/common/identifier.py b/heat/common/identifier.py index 5f5ae05204..fda926061a 100644 --- a/heat/common/identifier.py +++ b/heat/common/identifier.py @@ -13,11 +13,10 @@ # under the License. import re -import urllib -import urlparse import collections from heat.openstack.common import strutils +from heat.openstack.common.py3kcompat import urlutils class HeatIdentifier(collections.Mapping): @@ -63,10 +62,10 @@ class HeatIdentifier(collections.Mapping): if fields[1] != 'openstack' or fields[2] != 'heat' or not path: raise ValueError(_('"%s" is not a valid Heat ARN') % arn) - return cls(urllib.unquote(fields[4]), - urllib.unquote(path.group(1)), - urllib.unquote(path.group(2)), - urllib.unquote(path.group(3))) + return cls(urlutils.unquote(fields[4]), + urlutils.unquote(path.group(1)), + urlutils.unquote(path.group(2)), + urlutils.unquote(path.group(3))) @classmethod def from_arn_url(cls, url): @@ -75,7 +74,7 @@ class HeatIdentifier(collections.Mapping): The URL is expected to contain a valid arn as part of the path ''' # Sanity check the URL - urlp = urlparse.urlparse(url) + urlp = urlutils.urlparse(url) if (urlp.scheme not in ('http', 'https') or not urlp.netloc or not urlp.path): raise ValueError(_('"%s" is not a valid URL') % url) @@ -87,7 +86,7 @@ class HeatIdentifier(collections.Mapping): raise ValueError(_('"%s" is not a valid ARN URL') % url) # the +1 is to skip the leading / url_arn = urlp.path[match.start() + 1:] - arn = urllib.unquote(url_arn) + arn = urlutils.unquote(url_arn) return cls.from_arn(arn) def arn(self): @@ -95,21 +94,21 @@ class HeatIdentifier(collections.Mapping): Return an ARN of the form: arn:openstack:heat:::stacks// ''' - return 'arn:openstack:heat::%s:%s' % (urllib.quote(self.tenant, ''), + return 'arn:openstack:heat::%s:%s' % (urlutils.quote(self.tenant, ''), self._tenant_path()) def arn_url_path(self): ''' Return an ARN quoted correctly for use in a URL ''' - return '/' + urllib.quote(self.arn(), '') + return '/' + urlutils.quote(self.arn(), '') def url_path(self): ''' Return a URL-encoded path segment of a URL in the form: /stacks// ''' - return '/'.join((urllib.quote(self.tenant, ''), self._tenant_path())) + return '/'.join((urlutils.quote(self.tenant, ''), self._tenant_path())) def _tenant_path(self): ''' @@ -117,10 +116,10 @@ class HeatIdentifier(collections.Mapping): in the form: stacks// ''' - return 'stacks/%s/%s%s' % (urllib.quote(self.stack_name, ''), - urllib.quote(self.stack_id, ''), - urllib.quote(strutils.safe_encode( - self.path))) + return 'stacks/%s/%s%s' % (urlutils.quote(self.stack_name, ''), + urlutils.quote(self.stack_id, ''), + urlutils.quote(strutils.safe_encode( + self.path))) def _path_components(self): '''Return a list of the path components.''' diff --git a/heat/common/urlfetch.py b/heat/common/urlfetch.py index 409fd48bcc..6194e5ba9a 100644 --- a/heat/common/urlfetch.py +++ b/heat/common/urlfetch.py @@ -19,8 +19,6 @@ Utility for fetching a resource (e.g. a template) from a URL. import requests from requests import exceptions -import urllib2 -import urlparse from oslo.config import cfg @@ -28,6 +26,7 @@ cfg.CONF.import_opt('max_template_size', 'heat.common.config') from heat.openstack.common import log as logging from heat.openstack.common.gettextutils import _ +from heat.openstack.common.py3kcompat import urlutils logger = logging.getLogger(__name__) @@ -43,15 +42,15 @@ def get(url, allowed_schemes=('http', 'https')): ''' logger.info(_('Fetching data from %s') % url) - components = urlparse.urlparse(url) + components = urlutils.urlparse(url) if components.scheme not in allowed_schemes: raise IOError(_('Invalid URL scheme %s') % components.scheme) if components.scheme == 'file': try: - return urllib2.urlopen(url).read() - except urllib2.URLError as uex: + return urlutils.urlopen(url).read() + except urlutils.URLError as uex: raise IOError(_('Failed to retrieve template: %s') % str(uex)) try: diff --git a/heat/engine/resources/nova_utils.py b/heat/engine/resources/nova_utils.py index 82c858017e..2de208fcbe 100644 --- a/heat/engine/resources/nova_utils.py +++ b/heat/engine/resources/nova_utils.py @@ -21,8 +21,6 @@ import json import os import pkgutil -from urlparse import urlparse - from oslo.config import cfg from heat.common import exception @@ -31,6 +29,8 @@ from heat.engine import scheduler from heat.openstack.common import log as logging from heat.openstack.common.gettextutils import _ from heat.openstack.common import uuidutils +from heat.openstack.common.py3kcompat import urlutils + logger = logging.getLogger(__name__) @@ -175,8 +175,8 @@ def build_userdata(resource, userdata=None, instance_user=None): # Create a boto config which the cfntools on the host use to know # where the cfn and cw API's are to be accessed - cfn_url = urlparse(cfg.CONF.heat_metadata_server_url) - cw_url = urlparse(cfg.CONF.heat_watch_server_url) + cfn_url = urlutils.urlparse(cfg.CONF.heat_metadata_server_url) + cw_url = urlutils.urlparse(cfg.CONF.heat_watch_server_url) is_secure = cfg.CONF.instance_connection_is_secure vcerts = cfg.CONF.instance_connection_https_validate_certificates boto_cfg = "\n".join(["[Boto]", diff --git a/heat/engine/resources/s3.py b/heat/engine/resources/s3.py index aad610f64e..40ecdcd923 100644 --- a/heat/engine/resources/s3.py +++ b/heat/engine/resources/s3.py @@ -13,11 +13,12 @@ # License for the specific language governing permissions and limitations # under the License. -from urlparse import urlparse - from heat.engine import clients from heat.engine import resource + from heat.openstack.common import log as logging +from heat.openstack.common.py3kcompat import urlutils + logger = logging.getLogger(__name__) @@ -126,7 +127,7 @@ class S3Bucket(resource.Resource): def _resolve_attribute(self, name): url = self.swift().get_auth()[0] - parsed = list(urlparse(url)) + parsed = list(urlutils.urlparse(url)) if name == 'DomainName': return parsed[1].split(':')[0] elif name == 'WebsiteURL': diff --git a/heat/engine/resources/swift.py b/heat/engine/resources/swift.py index f18f443775..e69702b3cb 100644 --- a/heat/engine/resources/swift.py +++ b/heat/engine/resources/swift.py @@ -13,13 +13,14 @@ # License for the specific language governing permissions and limitations # under the License. -from urlparse import urlparse - from heat.common import exception from heat.engine import resource -from heat.openstack.common import log as logging from heat.engine import clients +from heat.openstack.common import log as logging +from heat.openstack.common.py3kcompat import urlutils + + logger = logging.getLogger(__name__) @@ -112,7 +113,7 @@ class SwiftContainer(resource.Resource): def FnGetAtt(self, key): url, token_id = self.swift().get_auth() - parsed = list(urlparse(url)) + parsed = list(urlutils.urlparse(url)) if key == 'DomainName': return parsed[1].split(':')[0] elif key == 'WebsiteURL': diff --git a/heat/engine/signal_responder.py b/heat/engine/signal_responder.py index 6ce858e6bf..f02004ddc7 100644 --- a/heat/engine/signal_responder.py +++ b/heat/engine/signal_responder.py @@ -13,9 +13,6 @@ # License for the specific language governing permissions and limitations # under the License. -import urllib -import urlparse - from oslo.config import cfg from keystoneclient.contrib.ec2 import utils as ec2_utils @@ -27,6 +24,8 @@ from heat.engine import resource from heat.openstack.common import log from heat.openstack.common.gettextutils import _ +from heat.openstack.common.py3kcompat import urlutils + LOG = log.getLogger(__name__) SIGNAL_TYPES = ( @@ -85,7 +84,7 @@ class SignalResponder(resource.Resource): waitcond_url = cfg.CONF.heat_waitcondition_server_url signal_url = waitcond_url.replace('/waitcondition', signal_type) - host_url = urlparse.urlparse(signal_url) + host_url = urlutils.urlparse(signal_url) path = self.identifier().arn_url_path() credentials = self.keystone().get_ec2_keypair(self.resource_id) @@ -94,7 +93,7 @@ class SignalResponder(resource.Resource): # prcessing in the CFN API (ec2token.py) has an unquoted path, so we # need to calculate the signature with the path component unquoted, but # ensure the actual URL contains the quoted version... - unquoted_path = urllib.unquote(host_url.path + path) + unquoted_path = urlutils.unquote(host_url.path + path) request = {'host': host_url.netloc.lower(), 'verb': SIGNAL_VERB[signal_type], 'path': unquoted_path, @@ -108,7 +107,7 @@ class SignalResponder(resource.Resource): signer = ec2_utils.Ec2Signer(credentials.secret) request['params']['Signature'] = signer.generate(request) - qs = urllib.urlencode(request['params']) + qs = urlutils.urlencode(request['params']) url = "%s%s?%s" % (signal_url.lower(), path, qs) diff --git a/heat/tests/test_api_openstack_v1_views_views_common.py b/heat/tests/test_api_openstack_v1_views_views_common.py index f09f1f35fc..fa0654089a 100644 --- a/heat/tests/test_api_openstack_v1_views_views_common.py +++ b/heat/tests/test_api_openstack_v1_views_views_common.py @@ -13,10 +13,10 @@ # under the License. import mock -import urlparse from heat.tests.common import HeatTestCase from heat.api.openstack.v1.views import views_common +from heat.openstack.common.py3kcompat import urlutils class TestViewsCommon(HeatTestCase): @@ -74,10 +74,11 @@ class TestViewsCommon(HeatTestCase): next_link = filter(lambda link: link['rel'] == 'next', links).pop() url = next_link['href'] - query_string = urlparse.urlparse(url).query - params = urlparse.parse_qs(query_string) - self.assertEqual('2', params['limit'][0]) - self.assertEqual('bar', params['foo'][0]) + query_string = urlutils.urlparse(url).query + params = {} + params.update(urlutils.parse_qsl(query_string)) + self.assertEqual('2', params['limit']) + self.assertEqual('bar', params['foo']) def test_get_collection_links_handles_invalid_limits(self): self.setUpGetCollectionLinks() diff --git a/heat/tests/test_urlfetch.py b/heat/tests/test_urlfetch.py index 8f2829f549..420e54a535 100644 --- a/heat/tests/test_urlfetch.py +++ b/heat/tests/test_urlfetch.py @@ -15,7 +15,6 @@ import requests from requests import exceptions -import urllib2 import cStringIO from oslo.config import cfg @@ -23,6 +22,8 @@ from oslo.config import cfg from heat.common import urlfetch from heat.tests.common import HeatTestCase +from heat.openstack.common.py3kcompat import urlutils + class Response: def __init__(self, buf=''): @@ -51,8 +52,8 @@ class UrlFetchTest(HeatTestCase): data = '{ "foo": "bar" }' url = 'file:///etc/profile' - self.m.StubOutWithMock(urllib2, 'urlopen') - urllib2.urlopen(url).AndReturn(cStringIO.StringIO(data)) + self.m.StubOutWithMock(urlutils, 'urlopen') + urlutils.urlopen(url).AndReturn(cStringIO.StringIO(data)) self.m.ReplayAll() self.assertEqual(data, urlfetch.get(url, allowed_schemes=['file'])) @@ -61,8 +62,8 @@ class UrlFetchTest(HeatTestCase): def test_file_scheme_failure(self): url = 'file:///etc/profile' - self.m.StubOutWithMock(urllib2, 'urlopen') - urllib2.urlopen(url).AndRaise(urllib2.URLError('oops')) + self.m.StubOutWithMock(urlutils, 'urlopen') + urlutils.urlopen(url).AndRaise(urlutils.URLError('oops')) self.m.ReplayAll() self.assertRaises(IOError, urlfetch.get, url, allowed_schemes=['file']) diff --git a/heat/tests/v1_1/fakes.py b/heat/tests/v1_1/fakes.py index 3031b7d733..b027b77552 100644 --- a/heat/tests/v1_1/fakes.py +++ b/heat/tests/v1_1/fakes.py @@ -14,12 +14,13 @@ # limitations under the License. import httplib2 -import urlparse from novaclient import client as base_client from novaclient.v1_1 import client from heat.tests import fakes +from heat.openstack.common.py3kcompat import urlutils + class FakeClient(fakes.FakeClient, client.Client): @@ -45,7 +46,7 @@ class FakeHTTPClient(base_client.HTTPClient): assert 'body' in kwargs # Call the method - args = urlparse.parse_qsl(urlparse.urlparse(url)[4]) + args = urlutils.parse_qsl(urlutils.urlparse(url)[4]) kwargs.update(args) munged_url = url.rsplit('?', 1)[0] munged_url = munged_url.strip('/').replace('/', '_').replace('.', '_')