openstack/deb-murano
Code Issues Proposed changes

Py3 compatibility fixes

Browse Source 
With this commit most of the Python 3 compatibility
issues in murano-engine are resolved.

If run on yaql with https://review.openstack.org/#/c/286110/
fix all of the unit tests except for one success.
The only failing test is base64 encoding/decoding test
which require rethink of resource management to get away
from string types for binary content

Change-Id: Iee87d27fe4f04118202de07f376d41fbf2c90f54
This commit is contained in:
Stan Lagun
2016-02-29 21:00:32 +03:00
committed by Alexander Tivelkov
parent f36fe4929d
commit 46ea32f4e8
24 changed files with 88 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
murano/common/messaging/__init__.py
View File

@@ -1,20 +0,0 @@
# Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from message import Message
from mqclient import MqClient
from subscription import Subscription
__all__ = ['Message', 'Subscription', 'MqClient']

6
murano/common/messaging/mqclient.py
View File

@@ -17,7 +17,8 @@ import ssl as ssl_module
from eventlet import patcher
from oslo_serialization import jsonutils
from subscription import Subscription
from murano.common.messaging import subscription
kombu = patcher.import_patched('kombu')
@@ -100,4 +101,5 @@ class MqClient(object):
if not self._connected:
raise RuntimeError('Not connected to RabbitMQ')
return Subscription(self._connection, queue, prefetch_count)
return subscription.Subscription(
self._connection, queue, prefetch_count)

11
murano/dsl/dsl_exception.py
View File

@@ -14,6 +14,8 @@
import sys
import six
from murano.dsl.principal_objects import stack_trace
@@ -53,8 +55,13 @@ class MuranoPlException(Exception):
def from_python_exception(exception, context):
stacktrace = stack_trace.create_stack_trace(context)
exception_type = type(exception)
names = ['{0}.{1}'.format(exception_type.__module__,
exception_type.__name__)]
builtins_module = 'builtins' if six.PY3 else 'exceptions'
module = exception_type.__module__
if module == builtins_module:
names = [exception_type.__name__]
else:
names = ['{0}.{1}'.format(exception_type.__module__,
exception_type.__name__)]
result = MuranoPlException(
names, str(exception), stacktrace)

2
murano/dsl/executor.py
View File

@@ -194,7 +194,7 @@ class MuranoDslExecutor(object):
@staticmethod
def _canonize_parameters(arguments_scheme, args, kwargs):
arg_names = arguments_scheme.keys()
arg_names = list(arguments_scheme.keys())
parameter_values = utils.filter_parameters_dict(kwargs)
for i, arg in enumerate(args):
name = arg_names[i]

10
murano/dsl/helpers.py
View File

@@ -17,7 +17,7 @@ import contextlib
import functools
import inspect
import itertools
import string
import re
import sys
import uuid
@@ -267,7 +267,7 @@ def parse_version_spec(version_spec):
semantic_version.Spec('==' + str(version_spec)))
if not version_spec:
version_spec = '0'
version_spec = str(version_spec).translate(None, string.whitespace)
version_spec = re.sub('\s+', '', str(version_spec))
if version_spec[0].isdigit():
version_spec = '==' + str(version_spec)
version_spec = semantic_version.Spec(version_spec)
@@ -532,3 +532,9 @@ def instantiate(data, owner, object_store, context, scope_type,
data['?']['id'] = uuid.uuid4().hex
return object_store.load(data, owner, context)
def function(c):
if hasattr(c, 'im_func'):
return c.im_func
return c

2
murano/dsl/macros.py
View File

@@ -27,7 +27,7 @@ class CodeBlock(expressions.DslExpression):
def __init__(self, body):
if not isinstance(body, list):
body = [body]
self.code_block = map(expressions.parse_expression, body)
self.code_block = list(map(expressions.parse_expression, body))
def execute(self, context):
for expr in self.code_block:

3
murano/dsl/murano_method.py
View File

@@ -177,7 +177,8 @@ class MuranoMethodArgument(dsl_types.MuranoMethodArgument, typespec.Spec,
self.murano_method.name, self.name,
e.path, six.text_type(e))
six.reraise(exceptions.ContractViolationException,
msg, sys.exc_info()[2])
exceptions.ContractViolationException(msg),
sys.exc_info()[2])
@property
def murano_method(self):

3
murano/dsl/murano_property.py
View File

@@ -41,7 +41,8 @@ class MuranoProperty(dsl_types.MuranoProperty, typespec.Spec,
msg = u'[{0}.{1}{2}] {3}'.format(
self.declaring_type.name, self.name, e.path, six.text_type(e))
six.reraise(exceptions.ContractViolationException,
msg, sys.exc_info()[2])
exceptions.ContractViolationException(msg),
sys.exc_info()[2])
@property
def declaring_type(self):

2
murano/dsl/murano_type.py
View File

@@ -304,7 +304,7 @@ class MuranoClass(dsl_types.MuranoClass, MuranoType, dslmeta.MetaProvider):
}
for cls, parent in helpers.traverse(
((self, parent) for parent in self._parents),
lambda (c, p): ((p, anc) for anc in p.declared_parents)):
lambda cp: ((cp[1], anc) for anc in cp[1].declared_parents)):
if cls.package != parent.package:
requirement = cls.package.requirements[parent.package.name]
aggregation.setdefault(parent.package.name, set()).add(

13
murano/dsl/type_scheme.py
View File

@@ -12,8 +12,6 @@
# License for the specific language governing permissions and limitations
# under the License.
import sys
import six
from yaql.language import specs
from yaql.language import utils
@@ -244,7 +242,7 @@ class TypeScheme(object):
if len(spec) < 1:
return data
shift = 0
max_length = sys.maxint
max_length = -1
min_length = 0
if isinstance(spec[-1], int):
min_length = spec[-1]
@@ -254,10 +252,14 @@ class TypeScheme(object):
min_length = spec[-2]
shift += 1
if not min_length <= len(data) <= max_length:
if max_length >= 0 and not min_length <= len(data) <= max_length:
raise exceptions.ContractViolationException(
'Array length {0} is not within [{1}..{2}] range'.format(
len(data), min_length, max_length))
elif not min_length <= len(data):
raise exceptions.ContractViolationException(
'Array length {0} must not be less than {1}'.format(
len(data), min_length))
def map_func():
for index, item in enumerate(data):
@@ -284,7 +286,8 @@ class TypeScheme(object):
if isinstance(spec, dsl_types.YaqlExpression):
child_context[''] = data
try:
return spec(context=child_context)
result = spec(context=child_context)
return result
except exceptions.ContractViolationException as e:
e.path = path
raise

17
murano/dsl/yaql_integration.py
View File

@@ -150,10 +150,9 @@ def get_function_definition(func, murano_method, original_name):
name, cls.__name__)
body = func
cls = murano_method.declaring_type.extension_class
if helpers.inspect_is_method(cls, original_name):
body = func.im_func
if helpers.inspect_is_classmethod(cls, original_name):
body = func.im_func
if (helpers.inspect_is_method(cls, original_name) or
helpers.inspect_is_classmethod(cls, original_name)):
body = helpers.function(func)
fd = specs.get_function_definition(
body, convention=CONVENTION,
parameter_type_func=param_type_func)
@@ -276,11 +275,11 @@ def get_class_factory_definition(cls, murano_class):
with helpers.contextual(__context):
return helpers.evaluate(cls(*args, **kwargs), __context)
if hasattr(cls.__init__, 'im_func'):
if '__init__' in cls.__dict__:
fd = specs.get_function_definition(
cls.__init__.im_func,
helpers.function(cls.__init__),
parameter_type_func=lambda name: _infer_parameter_type(
name, cls.__init__.im_class.__name__),
name, cls.__name__),
convention=CONVENTION)
else:
fd = specs.get_function_definition(lambda self: None)
@@ -302,12 +301,12 @@ def filter_parameters(__fd, *args, **kwargs):
position_args += 1
args = args[:position_args]
kwargs = kwargs.copy()
for name in kwargs.keys():
for name in list(kwargs.keys()):
if not utils.is_keyword(name):
del kwargs[name]
if '**' not in __fd.parameters:
names = {p.alias or p.name for p in six.itervalues(__fd.parameters)}
for name in kwargs.keys():
for name in list(kwargs.keys()):
if name not in names:
del kwargs[name]
return args, kwargs

14
murano/engine/package_loader.py
View File

@@ -127,8 +127,9 @@ class ApiPackageLoader(package_loader.MuranoPackageLoader):
self._lock_usage(package_definition)
except LookupError:
exc_info = sys.exc_info()
raise (exceptions.NoPackageForClassFound(class_name),
None, exc_info[2])
six.reraise(exceptions.NoPackageForClassFound,
exceptions.NoPackageForClassFound(class_name),
exc_info[2])
return self._to_dsl_package(
self._get_package_by_definition(package_definition))
@@ -256,7 +257,9 @@ class ApiPackageLoader(package_loader.MuranoPackageLoader):
package_id, str(e)
)
exc_info = sys.exc_info()
six.reraise(pkg_exc.PackageLoadError(msg), None, exc_info[2])
six.reraise(pkg_exc.PackageLoadError,
pkg_exc.PackageLoadError(msg),
exc_info[2])
package_file = None
try:
with tempfile.NamedTemporaryFile(delete=False) as package_file:
@@ -278,7 +281,9 @@ class ApiPackageLoader(package_loader.MuranoPackageLoader):
except IOError:
msg = 'Unable to extract package data for %s' % package_id
exc_info = sys.exc_info()
raise pkg_exc.PackageLoadError(msg), None, exc_info[2]
six.reraise(pkg_exc.PackageLoadError,
pkg_exc.PackageLoadError(msg),
exc_info[2])
finally:
try:
if package_file:
@@ -296,7 +301,6 @@ class ApiPackageLoader(package_loader.MuranoPackageLoader):
if not package_directory:
return
pkg_ids_listed = set()
try:
pkg_ids_listed = set(os.listdir(package_directory))
except OSError:

4
murano/engine/system/agent.py
View File

@@ -25,7 +25,7 @@ import six
from yaql import specs
import murano.common.exceptions as exceptions
import murano.common.messaging as messaging
from murano.common.messaging import message
from murano.dsl import dsl
import murano.engine.system.common as common
@@ -75,7 +75,7 @@ class Agent(object):
'by the server configuration')
def _prepare_message(self, template, msg_id):
msg = messaging.Message()
msg = message.Message()
msg.body = template
msg.id = msg_id
return msg

4
murano/engine/system/common.py
View File

@@ -15,7 +15,7 @@
from oslo_config import cfg
import murano.common.messaging as messaging
from murano.common.messaging import mqclient
CONF = cfg.CONF
@@ -31,4 +31,4 @@ def create_rmq_client():
'ssl': rabbitmq.ssl,
'ca_certs': rabbitmq.ca_certs.strip() or None
}
return messaging.MqClient(**connection_params)
return mqclient.MqClient(**connection_params)

4
murano/engine/system/logger.py
View File

@@ -13,8 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import exceptions
from oslo_log import log as logging
from yaql.language import specs
from yaql.language import yaqltypes
@@ -98,7 +96,7 @@ class Logger(object):
"""
try:
message = format_function(message, *args, **kwargs)
except (exceptions.IndexError, exceptions.KeyError):
except (IndexError, KeyError):
# NOTE(akhivin): we do not want break program workflow
# even formatting parameters are incorrect
self._underlying_logger.warning(

3
murano/engine/system/net_explorer.py
View File

@@ -72,7 +72,8 @@ class NetworkExplorer(object):
if uuidutils.is_uuid_like(external_network) \
else {'name': external_network}
networks = self._client.list_networks(**kwargs).get('networks')
ext_nets = filter(lambda n: n['router:external'], networks)
ext_nets = list(filter(lambda n: n['router:external'],
networks))
if len(ext_nets) == 0:
raise KeyError('Router %s could not be created, '
'no external network found' % router_name)

6
murano/engine/system/yaql_functions.py
View File

@@ -133,7 +133,7 @@ def _int2base(x, base):
:param base: number base, max value is 36
:return: integer converted to the specified base
"""
digs = string.digits + string.lowercase
digs = string.digits + string.ascii_lowercase
if x < 0:
sign = -1
elif x == 0:
@@ -144,7 +144,7 @@ def _int2base(x, base):
digits = []
while x:
digits.append(digs[x % base])
x /= base
x //= base
if sign < 0:
digits.append('-')
digits.reverse()
@@ -163,7 +163,7 @@ def random_name():
counter = _random_string_counter or 1
# generate first 5 random chars
prefix = ''.join(random.choice(string.lowercase) for _ in range(5))
prefix = ''.join(random.choice(string.ascii_lowercase) for _ in range(5))
# convert timestamp to higher base to shorten hostname string
# (up to 8 chars)
timestamp = _int2base(int(time.time() * 1000), 36)[:8]

9
murano/packages/load_utils.py
View File

@@ -19,6 +19,7 @@ import sys
import tempfile
import zipfile
import six
import yaml
from murano.common.plugins import package_types_loader
@@ -91,9 +92,11 @@ def load_from_dir(source_directory, filename='manifest.yaml'):
content = yaml.safe_load(stream)
except Exception as ex:
trace = sys.exc_info()[2]
raise e.PackageLoadError(
"Unable to load due to '{0}'".format(str(ex))), None, trace
if content:
six.reraise(
e.PackageLoadError,
e.PackageLoadError("Unable to load due to '{0}'".format(ex)),
trace)
else:
format_spec = str(content.get('Format') or 'MuranoPL/1.0')
if format_spec[0].isdigit():
format_spec = 'MuranoPL/' + format_spec

4
murano/packages/mpl_package.py
View File

@@ -37,7 +37,7 @@ class MuranoPlPackage(package_base.PackageBase):
full_path = os.path.join(self._source_directory, 'UI', self._ui_file)
if not os.path.isfile(full_path):
return None
with open(full_path) as stream:
with open(full_path, 'rb') as stream:
return stream.read()
@property
@@ -53,7 +53,7 @@ class MuranoPlPackage(package_base.PackageBase):
if not os.path.isfile(full_path):
raise exceptions.PackageClassLoadError(
name, 'File with class definition not found')
with open(full_path) as stream:
with open(full_path, 'rb') as stream:
return stream.read(), full_path
@property

7
murano/packages/package_base.py
View File

@@ -20,6 +20,7 @@ import re
import sys
import semantic_version
import six
from murano.packages import exceptions
from murano.packages import package
@@ -135,8 +136,10 @@ class PackageBase(package.Package):
return stream.read()
except Exception as ex:
trace = sys.exc_info()[2]
raise exceptions.PackageLoadError(
'Unable to load {0}: {1}'.format(what_image, ex)), None, trace
six.reraise(exceptions.PackageLoadError,
exceptions.PackageLoadError(
'Unable to load {0}: {1}'.format(what_image, ex)),
trace)
@staticmethod
def _check_full_name(full_name):

6
murano/tests/unit/dsl/foundation/runner.py
View File

@@ -105,8 +105,10 @@ class Runner(object):
original_exception, dsl_exception.MuranoPlException):
exc_traceback = getattr(
e, 'original_traceback', None) or sys.exc_info()[2]
raise type(original_exception), original_exception, \
exc_traceback
six.reraise(
type(original_exception),
original_exception,
exc_traceback)
raise
def __getattr__(self, item):

2
murano/tests/unit/dsl/foundation/test_package_loader.py
View File

@@ -88,7 +88,7 @@ class TestPackageLoader(package_loader.MuranoPackageLoader):
self._load_classes(class_def_file)
def _load_classes(self, class_def_file):
with open(class_def_file) as stream:
with open(class_def_file, 'rb') as stream:
data_lst = self._yaml_loader(stream.read(), class_def_file)
last_ns = {}

12
murano/tests/unit/dsl/test_contracts.py
View File

@@ -67,29 +67,29 @@ class TestContracts(test_case.DslTestCase):
def test_bool_contract(self):
result = self._runner.testBoolContract(True)
self.assertIsInstance(result, bool)
self.assertIs(result, True)
self.assertTrue(result)
result = self._runner.testBoolContract(False)
self.assertIsInstance(result, bool)
self.assertIs(result, False)
self.assertFalse(result)
def test_bool_from_int_contract(self):
result = self._runner.testBoolContract(10)
self.assertIsInstance(result, bool)
self.assertIs(result, True)
self.assertTrue(result)
result = self._runner.testBoolContract(0)
self.assertIsInstance(result, bool)
self.assertIs(result, False)
self.assertFalse(result)
def test_bool_from_string_contract(self):
result = self._runner.testBoolContract('something')
self.assertIsInstance(result, bool)
self.assertIs(result, True)
self.assertTrue(result)
result = self._runner.testBoolContract('')
self.assertIsInstance(result, bool)
self.assertIs(result, False)
self.assertFalse(result)
def test_bool_null_contract(self):
self.assertIsNone(self._runner.testIntContract(None))

2
murano/tests/unit/dsl/test_exceptions.py
View File

@@ -80,7 +80,7 @@ class TestExceptions(test_case.DslTestCase):
self.assertThat(
call_stack,
matchers.StartsWith(
'exceptions.LookupError: just random Python exception'))
'LookupError: just random Python exception'))
self.assertIsInstance(e.original_exception, LookupError)