Py3 compatibility fixes

With this commit most of the Python 3 compatibility
issues in murano-engine are resolved.

If run on yaql with https://review.openstack.org/#/c/286110/
fix all of the unit tests except for one success.
The only failing test is base64 encoding/decoding test
which require rethink of resource management to get away
from string types for binary content

Change-Id: Iee87d27fe4f04118202de07f376d41fbf2c90f54

murano/common/messaging/__init__.py

@@ -1,20 +0,0 @@
-# Copyright (c) 2013 Mirantis Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from message import Message
-from mqclient import MqClient
-from subscription import Subscription
-
-__all__ = ['Message', 'Subscription', 'MqClient']

murano/common/messaging/mqclient.py

@@ -17,7 +17,8 @@ import ssl as ssl_module
 
 from eventlet import patcher
 from oslo_serialization import jsonutils
-from subscription import Subscription
+
+from murano.common.messaging import subscription
 
 kombu = patcher.import_patched('kombu')
 
@@ -100,4 +101,5 @@ class MqClient(object):
         if not self._connected:
             raise RuntimeError('Not connected to RabbitMQ')
 
-        return Subscription(self._connection, queue, prefetch_count)
+        return subscription.Subscription(
+            self._connection, queue, prefetch_count)

murano/dsl/dsl_exception.py

@@ -14,6 +14,8 @@
 
 import sys
 
+import six
+
 from murano.dsl.principal_objects import stack_trace
 
 
@@ -53,8 +55,13 @@ class MuranoPlException(Exception):
     def from_python_exception(exception, context):
         stacktrace = stack_trace.create_stack_trace(context)
         exception_type = type(exception)
-        names = ['{0}.{1}'.format(exception_type.__module__,
+        builtins_module = 'builtins' if six.PY3 else 'exceptions'
-                                  exception_type.__name__)]
+        module = exception_type.__module__
+        if module == builtins_module:
+            names = [exception_type.__name__]
+        else:
+            names = ['{0}.{1}'.format(exception_type.__module__,
+                                      exception_type.__name__)]
 
         result = MuranoPlException(
             names, str(exception), stacktrace)

murano/dsl/executor.py

@@ -194,7 +194,7 @@ class MuranoDslExecutor(object):
 
     @staticmethod
     def _canonize_parameters(arguments_scheme, args, kwargs):
-        arg_names = arguments_scheme.keys()
+        arg_names = list(arguments_scheme.keys())
         parameter_values = utils.filter_parameters_dict(kwargs)
         for i, arg in enumerate(args):
             name = arg_names[i]

murano/dsl/helpers.py

@@ -17,7 +17,7 @@ import contextlib
 import functools
 import inspect
 import itertools
-import string
+import re
 import sys
 import uuid
 
@@ -267,7 +267,7 @@ def parse_version_spec(version_spec):
             semantic_version.Spec('==' + str(version_spec)))
     if not version_spec:
         version_spec = '0'
-    version_spec = str(version_spec).translate(None, string.whitespace)
+    version_spec = re.sub('\s+', '', str(version_spec))
     if version_spec[0].isdigit():
         version_spec = '==' + str(version_spec)
     version_spec = semantic_version.Spec(version_spec)
@@ -532,3 +532,9 @@ def instantiate(data, owner, object_store, context, scope_type,
         data['?']['id'] = uuid.uuid4().hex
 
     return object_store.load(data, owner, context)
+
+
+def function(c):
+    if hasattr(c, 'im_func'):
+        return c.im_func
+    return c

murano/dsl/macros.py

@@ -27,7 +27,7 @@ class CodeBlock(expressions.DslExpression):
     def __init__(self, body):
         if not isinstance(body, list):
             body = [body]
-        self.code_block = map(expressions.parse_expression, body)
+        self.code_block = list(map(expressions.parse_expression, body))
 
     def execute(self, context):
         for expr in self.code_block:

murano/dsl/murano_method.py

@@ -177,7 +177,8 @@ class MuranoMethodArgument(dsl_types.MuranoMethodArgument, typespec.Spec,
                 self.murano_method.name, self.name,
                 e.path, six.text_type(e))
             six.reraise(exceptions.ContractViolationException,
-                        msg, sys.exc_info()[2])
+                        exceptions.ContractViolationException(msg),
+                        sys.exc_info()[2])
 
     @property
     def murano_method(self):

murano/dsl/murano_property.py

@@ -41,7 +41,8 @@ class MuranoProperty(dsl_types.MuranoProperty, typespec.Spec,
             msg = u'[{0}.{1}{2}] {3}'.format(
                 self.declaring_type.name, self.name, e.path, six.text_type(e))
             six.reraise(exceptions.ContractViolationException,
-                        msg, sys.exc_info()[2])
+                        exceptions.ContractViolationException(msg),
+                        sys.exc_info()[2])
 
     @property
     def declaring_type(self):

murano/dsl/murano_type.py

@@ -304,7 +304,7 @@ class MuranoClass(dsl_types.MuranoClass, MuranoType, dslmeta.MetaProvider):
         }
         for cls, parent in helpers.traverse(
                 ((self, parent) for parent in self._parents),
-                lambda (c, p): ((p, anc) for anc in p.declared_parents)):
+                lambda cp: ((cp[1], anc) for anc in cp[1].declared_parents)):
             if cls.package != parent.package:
                 requirement = cls.package.requirements[parent.package.name]
                 aggregation.setdefault(parent.package.name, set()).add(

murano/dsl/type_scheme.py

@@ -12,8 +12,6 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import sys
-
 import six
 from yaql.language import specs
 from yaql.language import utils
@@ -244,7 +242,7 @@ class TypeScheme(object):
         if len(spec) < 1:
             return data
         shift = 0
-        max_length = sys.maxint
+        max_length = -1
         min_length = 0
         if isinstance(spec[-1], int):
             min_length = spec[-1]
@@ -254,10 +252,14 @@ class TypeScheme(object):
             min_length = spec[-2]
             shift += 1
 
-        if not min_length <= len(data) <= max_length:
+        if max_length >= 0 and not min_length <= len(data) <= max_length:
             raise exceptions.ContractViolationException(
                 'Array length {0} is not within [{1}..{2}] range'.format(
                     len(data), min_length, max_length))
+        elif not min_length <= len(data):
+            raise exceptions.ContractViolationException(
+                'Array length {0} must not be less than {1}'.format(
+                    len(data), min_length))
 
         def map_func():
             for index, item in enumerate(data):
@@ -284,7 +286,8 @@ class TypeScheme(object):
         if isinstance(spec, dsl_types.YaqlExpression):
             child_context[''] = data
             try:
-                return spec(context=child_context)
+                result = spec(context=child_context)
+                return result
             except exceptions.ContractViolationException as e:
                 e.path = path
                 raise

murano/dsl/yaql_integration.py

@@ -150,10 +150,9 @@ def get_function_definition(func, murano_method, original_name):
         name, cls.__name__)
     body = func
     cls = murano_method.declaring_type.extension_class
-    if helpers.inspect_is_method(cls, original_name):
+    if (helpers.inspect_is_method(cls, original_name) or
-        body = func.im_func
+            helpers.inspect_is_classmethod(cls, original_name)):
-    if helpers.inspect_is_classmethod(cls, original_name):
+        body = helpers.function(func)
-        body = func.im_func
     fd = specs.get_function_definition(
         body, convention=CONVENTION,
         parameter_type_func=param_type_func)
@@ -276,11 +275,11 @@ def get_class_factory_definition(cls, murano_class):
         with helpers.contextual(__context):
             return helpers.evaluate(cls(*args, **kwargs), __context)
 
-    if hasattr(cls.__init__, 'im_func'):
+    if '__init__' in cls.__dict__:
         fd = specs.get_function_definition(
-            cls.__init__.im_func,
+            helpers.function(cls.__init__),
             parameter_type_func=lambda name: _infer_parameter_type(
-                name, cls.__init__.im_class.__name__),
+                name, cls.__name__),
             convention=CONVENTION)
     else:
         fd = specs.get_function_definition(lambda self: None)
@@ -302,12 +301,12 @@ def filter_parameters(__fd, *args, **kwargs):
                 position_args += 1
         args = args[:position_args]
     kwargs = kwargs.copy()
-    for name in kwargs.keys():
+    for name in list(kwargs.keys()):
         if not utils.is_keyword(name):
             del kwargs[name]
     if '**' not in __fd.parameters:
         names = {p.alias or p.name for p in six.itervalues(__fd.parameters)}
-        for name in kwargs.keys():
+        for name in list(kwargs.keys()):
             if name not in names:
                 del kwargs[name]
     return args, kwargs

murano/engine/package_loader.py

@@ -127,8 +127,9 @@ class ApiPackageLoader(package_loader.MuranoPackageLoader):
             self._lock_usage(package_definition)
         except LookupError:
             exc_info = sys.exc_info()
-            raise (exceptions.NoPackageForClassFound(class_name),
+            six.reraise(exceptions.NoPackageForClassFound,
-                   None, exc_info[2])
+                        exceptions.NoPackageForClassFound(class_name),
+                        exc_info[2])
         return self._to_dsl_package(
             self._get_package_by_definition(package_definition))
 
@@ -256,7 +257,9 @@ class ApiPackageLoader(package_loader.MuranoPackageLoader):
                     package_id, str(e)
                 )
                 exc_info = sys.exc_info()
-                six.reraise(pkg_exc.PackageLoadError(msg), None, exc_info[2])
+                six.reraise(pkg_exc.PackageLoadError,
+                            pkg_exc.PackageLoadError(msg),
+                            exc_info[2])
             package_file = None
             try:
                 with tempfile.NamedTemporaryFile(delete=False) as package_file:
@@ -278,7 +281,9 @@ class ApiPackageLoader(package_loader.MuranoPackageLoader):
             except IOError:
                 msg = 'Unable to extract package data for %s' % package_id
                 exc_info = sys.exc_info()
-                raise pkg_exc.PackageLoadError(msg), None, exc_info[2]
+                six.reraise(pkg_exc.PackageLoadError,
+                            pkg_exc.PackageLoadError(msg),
+                            exc_info[2])
             finally:
                 try:
                     if package_file:
@@ -296,7 +301,6 @@ class ApiPackageLoader(package_loader.MuranoPackageLoader):
         if not package_directory:
             return
 
-        pkg_ids_listed = set()
         try:
             pkg_ids_listed = set(os.listdir(package_directory))
         except OSError:

murano/engine/system/agent.py

@@ -25,7 +25,7 @@ import six
 from yaql import specs
 
 import murano.common.exceptions as exceptions
-import murano.common.messaging as messaging
+from murano.common.messaging import message
 from murano.dsl import dsl
 import murano.engine.system.common as common
 
@@ -75,7 +75,7 @@ class Agent(object):
                 'by the server configuration')
 
     def _prepare_message(self, template, msg_id):
-        msg = messaging.Message()
+        msg = message.Message()
         msg.body = template
         msg.id = msg_id
         return msg

murano/engine/system/common.py

@@ -15,7 +15,7 @@
 
 from oslo_config import cfg
 
-import murano.common.messaging as messaging
+from murano.common.messaging import mqclient
 
 CONF = cfg.CONF
 
@@ -31,4 +31,4 @@ def create_rmq_client():
         'ssl': rabbitmq.ssl,
         'ca_certs': rabbitmq.ca_certs.strip() or None
     }
-    return messaging.MqClient(**connection_params)
+    return mqclient.MqClient(**connection_params)

murano/engine/system/logger.py

@@ -13,8 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import exceptions
-
 from oslo_log import log as logging
 from yaql.language import specs
 from yaql.language import yaqltypes
@@ -98,7 +96,7 @@ class Logger(object):
         """
         try:
             message = format_function(message, *args, **kwargs)
-        except (exceptions.IndexError, exceptions.KeyError):
+        except (IndexError, KeyError):
             # NOTE(akhivin): we do not want break program workflow
             # even formatting parameters are incorrect
             self._underlying_logger.warning(

murano/engine/system/net_explorer.py

@@ -72,7 +72,8 @@ class NetworkExplorer(object):
                     if uuidutils.is_uuid_like(external_network) \
                     else {'name': external_network}
                 networks = self._client.list_networks(**kwargs).get('networks')
-                ext_nets = filter(lambda n: n['router:external'], networks)
+                ext_nets = list(filter(lambda n: n['router:external'],
+                                       networks))
                 if len(ext_nets) == 0:
                     raise KeyError('Router %s could not be created, '
                                    'no external network found' % router_name)

murano/engine/system/yaql_functions.py

@@ -133,7 +133,7 @@ def _int2base(x, base):
     :param base: number base, max value is 36
     :return: integer converted to the specified base
     """
-    digs = string.digits + string.lowercase
+    digs = string.digits + string.ascii_lowercase
     if x < 0:
         sign = -1
     elif x == 0:
@@ -144,7 +144,7 @@ def _int2base(x, base):
     digits = []
     while x:
         digits.append(digs[x % base])
-        x /= base
+        x //= base
     if sign < 0:
         digits.append('-')
     digits.reverse()
@@ -163,7 +163,7 @@ def random_name():
 
     counter = _random_string_counter or 1
     # generate first 5 random chars
-    prefix = ''.join(random.choice(string.lowercase) for _ in range(5))
+    prefix = ''.join(random.choice(string.ascii_lowercase) for _ in range(5))
     # convert timestamp to higher base to shorten hostname string
     # (up to 8 chars)
     timestamp = _int2base(int(time.time() * 1000), 36)[:8]

murano/packages/load_utils.py

@@ -19,6 +19,7 @@ import sys
 import tempfile
 import zipfile
 
+import six
 import yaml
 
 from murano.common.plugins import package_types_loader
@@ -91,9 +92,11 @@ def load_from_dir(source_directory, filename='manifest.yaml'):
             content = yaml.safe_load(stream)
     except Exception as ex:
         trace = sys.exc_info()[2]
-        raise e.PackageLoadError(
+        six.reraise(
-            "Unable to load due to '{0}'".format(str(ex))), None, trace
+            e.PackageLoadError,
-    if content:
+            e.PackageLoadError("Unable to load due to '{0}'".format(ex)),
+            trace)
+    else:
         format_spec = str(content.get('Format') or 'MuranoPL/1.0')
         if format_spec[0].isdigit():
             format_spec = 'MuranoPL/' + format_spec

murano/packages/mpl_package.py

@@ -37,7 +37,7 @@ class MuranoPlPackage(package_base.PackageBase):
         full_path = os.path.join(self._source_directory, 'UI', self._ui_file)
         if not os.path.isfile(full_path):
             return None
-        with open(full_path) as stream:
+        with open(full_path, 'rb') as stream:
             return stream.read()
 
     @property
@@ -53,7 +53,7 @@ class MuranoPlPackage(package_base.PackageBase):
         if not os.path.isfile(full_path):
             raise exceptions.PackageClassLoadError(
                 name, 'File with class definition not found')
-        with open(full_path) as stream:
+        with open(full_path, 'rb') as stream:
             return stream.read(), full_path
 
     @property

murano/packages/package_base.py

@@ -20,6 +20,7 @@ import re
 import sys
 
 import semantic_version
+import six
 
 from murano.packages import exceptions
 from murano.packages import package
@@ -135,8 +136,10 @@ class PackageBase(package.Package):
                 return stream.read()
         except Exception as ex:
             trace = sys.exc_info()[2]
-            raise exceptions.PackageLoadError(
+            six.reraise(exceptions.PackageLoadError,
-                'Unable to load {0}: {1}'.format(what_image, ex)), None, trace
+                        exceptions.PackageLoadError(
+                            'Unable to load {0}: {1}'.format(what_image, ex)),
+                        trace)
 
     @staticmethod
     def _check_full_name(full_name):

murano/tests/unit/dsl/foundation/runner.py

@@ -105,8 +105,10 @@ class Runner(object):
                         original_exception, dsl_exception.MuranoPlException):
                     exc_traceback = getattr(
                         e, 'original_traceback', None) or sys.exc_info()[2]
-                    raise type(original_exception), original_exception, \
+                    six.reraise(
-                        exc_traceback
+                        type(original_exception),
+                        original_exception,
+                        exc_traceback)
             raise
 
     def __getattr__(self, item):

murano/tests/unit/dsl/foundation/test_package_loader.py

@@ -88,7 +88,7 @@ class TestPackageLoader(package_loader.MuranoPackageLoader):
             self._load_classes(class_def_file)
 
     def _load_classes(self, class_def_file):
-        with open(class_def_file) as stream:
+        with open(class_def_file, 'rb') as stream:
             data_lst = self._yaml_loader(stream.read(), class_def_file)
 
         last_ns = {}

murano/tests/unit/dsl/test_contracts.py

@@ -67,29 +67,29 @@ class TestContracts(test_case.DslTestCase):
     def test_bool_contract(self):
         result = self._runner.testBoolContract(True)
         self.assertIsInstance(result, bool)
-        self.assertIs(result, True)
+        self.assertTrue(result)
 
         result = self._runner.testBoolContract(False)
         self.assertIsInstance(result, bool)
-        self.assertIs(result, False)
+        self.assertFalse(result)
 
     def test_bool_from_int_contract(self):
         result = self._runner.testBoolContract(10)
         self.assertIsInstance(result, bool)
-        self.assertIs(result, True)
+        self.assertTrue(result)
 
         result = self._runner.testBoolContract(0)
         self.assertIsInstance(result, bool)
-        self.assertIs(result, False)
+        self.assertFalse(result)
 
     def test_bool_from_string_contract(self):
         result = self._runner.testBoolContract('something')
         self.assertIsInstance(result, bool)
-        self.assertIs(result, True)
+        self.assertTrue(result)
 
         result = self._runner.testBoolContract('')
         self.assertIsInstance(result, bool)
-        self.assertIs(result, False)
+        self.assertFalse(result)
 
     def test_bool_null_contract(self):
         self.assertIsNone(self._runner.testIntContract(None))

murano/tests/unit/dsl/test_exceptions.py

@@ -80,7 +80,7 @@ class TestExceptions(test_case.DslTestCase):
         self.assertThat(
             call_stack,
             matchers.StartsWith(
-                'exceptions.LookupError: just random Python exception'))
+                'LookupError: just random Python exception'))
 
         self.assertIsInstance(e.original_exception, LookupError)