openstack/heat
Code Issues Proposed changes

Remove six and python 2.7 full support

Browse Source 
Six is in use to help us to keep support for python 2.7.
Since the ussuri cycle we decide to remove the python 2.7
support so we can go ahead and also remove six usage from
the python code.

Review process and help
-----------------------
Removing six introduce a lot of changes and an huge amount of modified files
To simplify reviews we decided to split changes into several patches to avoid
painful reviews and avoid mistakes.

To review this patch you can use the six documentation [1] to obtain help and
understand choices.

Additional informations
-----------------------
Changes related to 'six.b(data)' [2]
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

six.b [2] encode the given datas in latin-1 in python3 so I did the same
things in this patch.

Latin-1 is equal to iso-8859-1 [3].

This encoding is the default encoding [4] of certain descriptive HTTP
headers.

I suggest to keep latin-1 for the moment and to move to another encoding
in a follow-up patch if needed to move to most powerful encoding (utf8).

HTML4 support utf8 charset and utf8 is the default charset for HTML5 [5].

Note that this commit message is autogenerated and not necesserly contains
changes related to 'six.b'

[1] https://six.readthedocs.io/
[2] https://six.readthedocs.io/#six.b
[3] https://docs.python.org/3/library/codecs.html#standard-encodings
[4] https://www.w3schools.com/charsets/ref_html_8859.asp
[5] https://www.w3schools.com/html/html_charset.asp

Patch 4 of a serie of 28 patches

Change-Id: I871c2dad10abc35790e730c7c4c5272f499b7623
This commit is contained in:
Hervé Beraud 2019-11-20 19:37:26 +01:00
parent fccd312871
commit 5877da06a1
11 changed files with 55 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
heat/common/template_format.py
View File

@ -15,7 +15,6 @@ import collections
from oslo_config import cfg
from oslo_serialization import jsonutils
import six
import yaml
from heat.common import exception
@ -73,7 +72,7 @@ def simple_parse(tmpl_str, tmpl_url=None):
except yaml.YAMLError as yea:
if tmpl_url is None:
tmpl_url = '[root stack]'
yea = six.text_type(yea)
yea = str(yea)
msg = _('Error parsing template %(tmpl)s '
'%(yea)s') % {'tmpl': tmpl_url, 'yea': yea}
raise ValueError(msg)
@ -111,7 +110,7 @@ def parse(tmpl_str, tmpl_url=None):
# TODO(ricolin): Move this validation to api side.
# Validate nested stack template.
validate_template_limit(six.text_type(tmpl_str))
validate_template_limit(str(tmpl_str))
tpl = simple_parse(tmpl_str, tmpl_url)
# Looking for supported version keys in the loaded template
@ -136,7 +135,7 @@ def convert_json_to_yaml(json_str):
def top_level_items(tpl):
yield ("HeatTemplateFormatVersion", '2012-12-12')
for k, v in six.iteritems(tpl):
for k, v in tpl.items():
if k != 'AWSTemplateFormatVersion':
yield k, v

4
heat/common/urlfetch.py
View File

@ -14,12 +14,14 @@
"""Utility for fetching a resource (e.g. a template) from a URL."""
import socket
import urllib.error
import urllib.parse
import urllib.request
from oslo_config import cfg
from oslo_log import log as logging
import requests
from requests import exceptions
from six.moves import urllib
from heat.common import exception
from heat.common.i18n import _

8
heat/common/wsgi.py
View File

@ -40,7 +40,6 @@ from oslo_utils import encodeutils
from oslo_utils import importutils
from paste.deploy import loadwsgi
from routes import middleware
import six
import webob.dec
import webob.exc
@ -648,7 +647,7 @@ class Debug(Middleware):
resp = req.get_response(self.application)
print(("*" * 40) + " RESPONSE HEADERS")
for (key, value) in six.iteritems(resp.headers):
for (key, value) in resp.headers.items():
print(key, "=", value)
print('')
@ -823,7 +822,7 @@ class JSONRequestDeserializer(object):
raise exception.RequestLimitExceeded(message=msg)
return jsonutils.loads(datastring)
except ValueError as ex:
raise webob.exc.HTTPBadRequest(six.text_type(ex))
raise webob.exc.HTTPBadRequest(str(ex))
def default(self, request):
if self.has_body(request):
@ -1004,8 +1003,7 @@ def translate_exception(exc, locale):
return exc
@six.add_metaclass(abc.ABCMeta)
class BasePasteFactory(object):
class BasePasteFactory(object, metaclass=abc.ABCMeta):
"""A base class for paste app and filter factories.
Sub-classes must override the KEY class attribute and provide

11
heat/db/sqlalchemy/api.py
View File

@ -28,7 +28,6 @@ from oslo_utils import encodeutils
from oslo_utils import excutils
from oslo_utils import timeutils
import osprofiler.sqlalchemy
import six
import sqlalchemy
from sqlalchemy import and_
from sqlalchemy import func
@ -112,7 +111,7 @@ def retry_on_deadlock(func):
def update_and_save(context, obj, values):
with context.session.begin(subtransactions=True):
for k, v in six.iteritems(values):
for k, v in values.items():
setattr(obj, k, v)
@ -663,7 +662,7 @@ def _get_sort_keys(sort_keys, mapping):
:param mapping: a mapping from keys to DB column names
:returns: filtered list of sort keys
"""
if isinstance(sort_keys, six.string_types):
if isinstance(sort_keys, str):
sort_keys = [sort_keys]
return [mapping[key] for key in sort_keys or [] if key in mapping]
@ -951,7 +950,7 @@ def user_creds_create(context):
else:
user_creds_ref.update(values)
method, password = crypt.encrypt(values['password'])
if len(six.text_type(password)) > 255:
if len(str(password)) > 255:
raise exception.Error(_("Length of OS_PASSWORD after encryption"
" exceeds Heat limit (255 chars)"))
user_creds_ref.password = password
@ -1177,7 +1176,7 @@ def event_create(context, values):
_delete_event_rows(context, values['stack_id'],
cfg.CONF.event_purge_batch_size)
except db_exception.DBError as exc:
LOG.error('Failed to purge events: %s', six.text_type(exc))
LOG.error('Failed to purge events: %s', str(exc))
event_ref = models.Event()
event_ref.update(values)
event_ref.save(context.session)
@ -1666,7 +1665,7 @@ def _db_encrypt_or_decrypt_template_params(
not param_schemata[param_name].hidden):
continue
encrypted_val = crypt.encrypt(
six.text_type(param_val), encryption_key)
str(param_val), encryption_key)
env['parameters'][param_name] = encrypted_val
encrypted_params.append(param_name)
needs_update = True

4
heat/db/sqlalchemy/filters.py
View File

@ -11,8 +11,6 @@
# License for the specific language governing permissions and limitations
# under the License.
import six
def exact_filter(query, model, filters):
"""Applies exact match filtering to a query.
@ -33,7 +31,7 @@ def exact_filter(query, model, filters):
if filters is None:
filters = {}
for key, value in six.iteritems(filters):
for key, value in filters.items():
if isinstance(value, (list, tuple, set, frozenset)):
column_attr = getattr(model, key)
query = query.filter(column_attr.in_(value))

7
heat/engine/api.py
View File

@ -15,7 +15,6 @@ import collections
from oslo_log import log as logging
from oslo_utils import timeutils
import six
from heat.common.i18n import _
from heat.common import param_utils
@ -65,7 +64,7 @@ def extract_args(params):
raise ValueError(_('Invalid tags, not a list: %s') % tags)
for tag in tags:
if not isinstance(tag, six.string_types):
if not isinstance(tag, str):
raise ValueError(_('Invalid tag, "%s" is not a string') % tag)
if len(tag) > 80:
@ -188,7 +187,7 @@ def format_stack_output(output_defn, resolve_value=True):
except Exception as ex:
# We don't need error raising, just adding output_error to
# resulting dict.
result.update({rpc_api.OUTPUT_ERROR: six.text_type(ex)})
result.update({rpc_api.OUTPUT_ERROR: str(ex)})
finally:
result.update({rpc_api.OUTPUT_VALUE: value})
@ -212,7 +211,7 @@ def format_stack(stack, preview=False, resolve_outputs=True):
rpc_api.STACK_UPDATED_TIME: updated_time,
rpc_api.STACK_DELETION_TIME: deleted_time,
rpc_api.STACK_NOTIFICATION_TOPICS: [], # TODO(therve) Not implemented
rpc_api.STACK_PARAMETERS: stack.parameters.map(six.text_type),
rpc_api.STACK_PARAMETERS: stack.parameters.map(str),
rpc_api.STACK_DESCRIPTION: stack.t[stack.t.DESCRIPTION],
rpc_api.STACK_TMPL_DESCRIPTION: stack.t[stack.t.DESCRIPTION],
rpc_api.STACK_CAPABILITIES: [], # TODO(?) Not implemented yet

12
heat/engine/attributes.py
View File

@ -12,9 +12,9 @@
# under the License.
import collections
import functools
from oslo_utils import strutils
import six
from heat.common.i18n import _
from heat.engine import constraints as constr
@ -206,14 +206,14 @@ class Attributes(collections.Mapping):
def _validate_type(self, attrib, value):
if attrib.schema.type == attrib.schema.STRING:
if not isinstance(value, six.string_types):
if not isinstance(value, str):
LOG.warning("Attribute %(name)s is not of type "
"%(att_type)s",
{'name': attrib.name,
'att_type': attrib.schema.STRING})
elif attrib.schema.type == attrib.schema.LIST:
if (not isinstance(value, collections.Sequence)
or isinstance(value, six.string_types)):
or isinstance(value, str)):
LOG.warning("Attribute %(name)s is not of type "
"%(att_type)s",
{'name': attrib.name,
@ -296,7 +296,7 @@ class Attributes(collections.Mapping):
def __repr__(self):
return ("Attributes for %s:\n\t" % self._resource_name +
'\n\t'.join(six.itervalues(self)))
'\n\t'.join(self.values()))
def select_from_attribute(attribute_value, path):
@ -311,12 +311,12 @@ def select_from_attribute(attribute_value, path):
collections.Sequence)):
raise TypeError(_("Can't traverse attribute path"))
if not isinstance(key, (six.string_types, int)):
if not isinstance(key, (str, int)):
raise TypeError(_('Path components in attributes must be strings'))
return collection[key]
try:
return six.moves.reduce(get_path_component, path, attribute_value)
return functools.reduce(get_path_component, path, attribute_value)
except (KeyError, IndexError, TypeError):
return None

29
heat/engine/cfn/functions.py
View File

@ -14,7 +14,6 @@
import collections
from oslo_serialization import jsonutils
import six
from heat.api.aws import utils as aws_utils
from heat.common import exception
@ -39,7 +38,7 @@ class FindInMap(function.Function):
try:
self._mapname, self._mapkey, self._mapvalue = self.args
except ValueError as ex:
raise KeyError(six.text_type(ex))
raise KeyError(str(ex))
def result(self):
mapping = self.stack.t.maps[function.resolve(self._mapname)]
@ -160,7 +159,7 @@ class Select(function.Function):
# Handle by returning an empty string
return ''
if isinstance(strings, six.string_types):
if isinstance(strings, str):
# might be serialized json.
try:
strings = jsonutils.loads(strings)
@ -170,7 +169,7 @@ class Select(function.Function):
raise ValueError(_('"%(fn_name)s": %(err)s') % fmt_data)
if isinstance(strings, collections.Mapping):
if not isinstance(index, six.string_types):
if not isinstance(index, str):
raise TypeError(_('Index to "%s" must be a string') %
self.fn_name)
return strings.get(index, '')
@ -181,8 +180,8 @@ class Select(function.Function):
pass
if (isinstance(strings, collections.Sequence) and
not isinstance(strings, six.string_types)):
if not isinstance(index, six.integer_types):
not isinstance(strings, str)):
if not isinstance(index, int):
raise TypeError(_('Index to "%s" must be an integer') %
self.fn_name)
@ -230,7 +229,7 @@ class Split(function.Function):
fmt_data = {'fn_name': self.fn_name,
'example': example}
if isinstance(self.args, (six.string_types, collections.Mapping)):
if isinstance(self.args, (str, collections.Mapping)):
raise TypeError(_('Incorrect arguments to "%(fn_name)s" '
'should be: %(example)s') % fmt_data)
@ -243,10 +242,10 @@ class Split(function.Function):
def result(self):
strings = function.resolve(self._strings)
if not isinstance(self._delim, six.string_types):
if not isinstance(self._delim, str):
raise TypeError(_("Delimiter for %s must be string") %
self.fn_name)
if not isinstance(strings, six.string_types):
if not isinstance(strings, str):
raise TypeError(_("String to split must be string; got %s") %
type(strings))
@ -279,7 +278,7 @@ class Replace(hot_funcs.Replace):
fmt_data = {'fn_name': self.fn_name,
'example': example}
if isinstance(self.args, (six.string_types, collections.Mapping)):
if isinstance(self.args, (str, collections.Mapping)):
raise TypeError(_('Incorrect arguments to "%(fn_name)s" '
'should be: %(example)s') % fmt_data)
@ -306,7 +305,7 @@ class Base64(function.Function):
def result(self):
resolved = function.resolve(self.args)
if not isinstance(resolved, six.string_types):
if not isinstance(resolved, str):
raise TypeError(_('"%s" argument must be a string') % self.fn_name)
return resolved
@ -342,10 +341,10 @@ class MemberListToMap(function.Function):
'''
raise TypeError(_('Wrong Arguments try: "%s"') % correct)
if not isinstance(self._keyname, six.string_types):
if not isinstance(self._keyname, str):
raise TypeError(_('%s Key Name must be a string') % self.fn_name)
if not isinstance(self._valuename, six.string_types):
if not isinstance(self._valuename, str):
raise TypeError(_('%s Value Name must be a string') % self.fn_name)
def result(self):
@ -355,7 +354,7 @@ class MemberListToMap(function.Function):
raise TypeError(_('Member list must be a list'))
def item(s):
if not isinstance(s, six.string_types):
if not isinstance(s, str):
raise TypeError(_("Member list items must be strings"))
return s.split('=', 1)
@ -430,7 +429,7 @@ class Not(hot_funcs.Not):
'[condition]') % self.fn_name
if (not self.args or
not isinstance(self.args, collections.Sequence) or
isinstance(self.args, six.string_types)):
isinstance(self.args, str)):
raise ValueError(msg)
if len(self.args) != 1:
raise ValueError(msg)

12
heat/engine/cfn/template.py
View File

@ -13,8 +13,6 @@
import functools
import six
from heat.common import exception
from heat.common.i18n import _
from heat.engine.cfn import functions as cfn_funcs
@ -98,12 +96,12 @@ class CfnTemplateBase(template_common.CommonTemplate):
def param_schemata(self, param_defaults=None):
params = self.t.get(self.PARAMETERS) or {}
pdefaults = param_defaults or {}
for name, schema in six.iteritems(params):
for name, schema in params.items():
if name in pdefaults:
params[name][parameters.DEFAULT] = pdefaults[name]
return dict((name, parameters.Schema.from_dict(name, schema))
for name, schema in six.iteritems(params))
for name, schema in params.items())
def get_section_name(self, section):
return section
@ -124,7 +122,7 @@ class CfnTemplateBase(template_common.CommonTemplate):
defn_data = dict(self._rsrc_defn_args(stack, name,
snippet))
except (TypeError, ValueError, KeyError) as ex:
msg = six.text_type(ex)
msg = str(ex)
raise exception.StackValidationFailed(message=msg)
defn = rsrc_defn.ResourceDefinition(name, **defn_data)
@ -135,7 +133,7 @@ class CfnTemplateBase(template_common.CommonTemplate):
enabled = conditions.is_enabled(cond_name)
except ValueError as exc:
path = [self.RESOURCES, name, self.RES_CONDITION]
message = six.text_type(exc)
message = str(exc)
raise exception.StackValidationFailed(path=path,
message=message)
if not enabled:
@ -234,7 +232,7 @@ class CfnTemplate(CfnTemplateBase):
yield ('condition',
self._parse_resource_field(self.RES_CONDITION,
(six.string_types, bool,
(str, bool,
function.Function),
'string or boolean',
name, data, parse_cond))

8
heat/engine/check_resource.py
View File

@ -13,8 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import six
import eventlet.queue
import functools
@ -83,7 +81,7 @@ class CheckResource(object):
'during resource %s' % rsrc.action)
rsrc.state_set(rsrc.action,
rsrc.FAILED,
six.text_type(status_reason))
str(status_reason))
return True
elif (rs_obj.engine_id is None and
rs_obj.current_template_id == prev_template_id):
@ -177,7 +175,7 @@ class CheckResource(object):
except exception.ResourceFailure as ex:
action = ex.action or rsrc.action
reason = 'Resource %s failed: %s' % (action,
six.text_type(ex))
str(ex))
self._handle_resource_failure(cnxt, is_update, rsrc.id,
stack, reason)
except scheduler.Timeout:
@ -323,7 +321,7 @@ class CheckResource(object):
rsrc, stack)
except BaseException as exc:
with excutils.save_and_reraise_exception():
msg = six.text_type(exc)
msg = str(exc)
LOG.exception("Unexpected exception in resource check.")
self._handle_resource_failure(cnxt, is_update, rsrc.id,
stack, msg)

15
heat/tests/test_urlfetch.py
View File

@ -11,10 +11,13 @@
# License for the specific language governing permissions and limitations
# under the License.
import io
import urllib.error
import urllib.request
from oslo_config import cfg
import requests
from requests import exceptions
import six
from heat.common import urlfetch
from heat.tests import common
@ -42,15 +45,15 @@ class UrlFetchTest(common.HeatTestCase):
def test_file_scheme_supported(self):
data = '{ "foo": "bar" }'
url = 'file:///etc/profile'
mock_open = self.patchobject(six.moves.urllib.request, 'urlopen')
mock_open.return_value = six.moves.cStringIO(data)
mock_open = self.patchobject(urllib.request, 'urlopen')
mock_open.return_value = io.StringIO(data)
self.assertEqual(data, urlfetch.get(url, allowed_schemes=['file']))
mock_open.assert_called_once_with(url)
def test_file_scheme_failure(self):
url = 'file:///etc/profile'
mock_open = self.patchobject(six.moves.urllib.request, 'urlopen')
mock_open.side_effect = six.moves.urllib.error.URLError('oops')
mock_open = self.patchobject(urllib.request, 'urlopen')
mock_open.side_effect = urllib.error.URLError('oops')
self.assertRaises(urlfetch.URLFetchError,
urlfetch.get, url, allowed_schemes=['file'])
mock_open.assert_called_once_with(url)
@ -109,5 +112,5 @@ class UrlFetchTest(common.HeatTestCase):
mock_get.return_value = response
exception = self.assertRaises(urlfetch.URLFetchError,
urlfetch.get, url)
self.assertIn("Template exceeds", six.text_type(exception))
self.assertIn("Template exceeds", str(exception))
mock_get.assert_called_once_with(url, stream=True)