Support for Unicode strings in MuranoPL was fixed

There were several issues with Unicode support in MuranoPL
after switch to yaql 1.0. Each of those issues caused any deploy
to fail if it used to do anything with non-ASCII strings

Fixed issues are:
* Unicode strings should not be encoded to str types anymore.
yaql 1.0 has native support for the Unicode and fails when non-
ASCII chars encounter in str expressions. Also tests for Unicode
support are now part of the yaql

* Traces of execution were logged not as Unicode strings.  Because
those traces contain parameter values ant function return value
logging failed when any of above contained non-ASCII chars

* Stack trace logging failed when frame expression contained
non-ASCII chars

* Exception messages could not contain non-ASCII chars

Also Logging API was not Unicode ready

Change-Id: Ief0b45f15669c5f8ee74fd6ff41fa5bc39c9500b
Closes-Bug: #1494275

murano/dsl/dsl_exception.py

@@ -20,7 +20,7 @@ from murano.dsl.principal_objects import stack_trace
 class MuranoPlException(Exception):
     def __init__(self, names, message, stacktrace, extra=None, cause=None):
         super(MuranoPlException, self).__init__(
-            '[{0}]: {1}'.format(', '.join(names), message))
+            u'[{0}]: {1}'.format(', '.join(names), message))
         if not isinstance(names, list):
             names = [names]
         self._names = names

murano/dsl/executor.py

@@ -145,10 +145,10 @@ class MuranoDslExecutor(object):
     def _log_method(self, context, args, kwargs):
         method = helpers.get_current_method(context)
         param_gen = itertools.chain(
-            (str(arg) for arg in args),
+            (unicode(arg) for arg in args),
-            ('{0} => {1}'.format(name, value)
+            (u'{0} => {1}'.format(name, value)
              for name, value in kwargs.iteritems()))
-        params_str = ', '.join(param_gen)
+        params_str = u', '.join(param_gen)
         method_name = '{0}::{1}'.format(method.murano_class.name, method.name)
         thread_id = helpers.get_current_thread_id()
         caller_str = ''
@@ -158,18 +158,20 @@ class MuranoDslExecutor(object):
             if frame['location']:
                 caller_str = ' called from ' + stack_trace.format_frame(frame)
 
-        LOG.trace('{thread}: Begin execution {method}({params}){caller}'
+        LOG.trace(u'{thread}: Begin execution {method}({params}){caller}'
                   .format(thread=thread_id, method=method_name,
                           params=params_str, caller=caller_str))
         try:
             def log_result(result):
-                LOG.trace('{thread}: End execution {method} with result '
+                LOG.trace(
-                          '{result}'.format(thread=thread_id,
+                    u'{thread}: End execution {method} with result '
-                                            method=method_name, result=result))
+                    u'{result}'.format(
+                        thread=thread_id, method=method_name, result=result))
             yield log_result
         except Exception as e:
-            LOG.trace('{thread}: End execution {method} with exception {exc}'
+            LOG.trace(
-                      .format(thread=thread_id, method=method_name, exc=e))
+                u'{thread}: End execution {method} with exception '
+                u'{exc}'.format(thread=thread_id, method=method_name, exc=e))
             raise
 
     @staticmethod

murano/dsl/principal_objects/stack_trace.py

@@ -66,7 +66,7 @@ def compose_stack_frame(context):
     method = helpers.get_current_method(context)
     return {
         'instruction': None if instruction is None
-        else str(instruction),
+        else unicode(instruction),
 
         'location': None if instruction is None
         else instruction.source_file_position,
@@ -94,10 +94,10 @@ def format_frame(frame, prefix=''):
             instruction,
             prefix
         )
-        return ('{5}File "{0}", line {1}{2} in method {3}\n'
+        return (u'{5}File "{0}", line {1}{2} in method {3}\n'
-                '{5}    {4}').format(*args)
+                u'{5}    {4}').format(*args)
     else:
-        return '{2}File <unknown> in method {0}\n{2}    {1}'.format(
+        return u'{2}File <unknown> in method {0}\n{2}    {1}'.format(
             method, instruction, prefix)
 
 

murano/dsl/virtual_exceptions.py

@@ -50,10 +50,10 @@ class ThrowMacro(expressions.DslExpression):
             helpers.evaluate(self._message, context),
             stacktrace, self._extra, cause)
 
-    def __str__(self):
+    def __unicode__(self):
         if self._message:
-            return 'Throw {0}: {1}'.format(self._names, self._message)
+            return u'Throw {0}: {1}'.format(self._names, self._message)
-        return 'Throw ' + str(self._names)
+        return u'Throw ' + unicode(self._names)
 
 
 class CatchBlock(expressions.DslExpression):

murano/dsl/yaql_expression.py

@@ -15,7 +15,6 @@
 import re
 import types
 
-from oslo_utils import encodeutils
 from yaql.language import exceptions as yaql_exceptions
 from yaql.language import expressions
 
@@ -28,7 +27,7 @@ class YaqlExpression(dsl_types.YaqlExpression):
     def __init__(self, expression, version):
         self._version = version
         if isinstance(expression, types.StringTypes):
-            self._expression = encodeutils.safe_encode(expression)
+            self._expression = unicode(expression)
             self._parsed_expression = yaql_integration.parse(
                 self._expression, version)
             self._file_position = None

murano/engine/system/logger.py

@@ -21,7 +21,7 @@ from yaql.language import yaqltypes
 
 from murano.dsl import dsl
 
-NAME_TEMPLATE = 'applications.{0}'
+NAME_TEMPLATE = u'applications.{0}'
 
 inject_format = specs.inject(
     '_Logger__yaql_format_function',
@@ -39,42 +39,49 @@ class Logger(object):
         self._underlying_logger = logging.getLogger(
             NAME_TEMPLATE.format(logger_name))
 
+    @specs.parameter('_Logger__message', yaqltypes.String())
     @inject_format
     def trace(__self, __yaql_format_function, __message, *args, **kwargs):
         __self._log(__self._underlying_logger.trace,
                     __yaql_format_function, __message, args, kwargs)
 
+    @specs.parameter('_Logger__message', yaqltypes.String())
     @inject_format
     def debug(__self, __yaql_format_function, __message, *args, **kwargs):
         __self._log(__self._underlying_logger.debug,
                     __yaql_format_function, __message, args, kwargs)
 
+    @specs.parameter('_Logger__message', yaqltypes.String())
     @inject_format
     def info(__self, __yaql_format_function, __message, *args, **kwargs):
         __self._log(__self._underlying_logger.info,
                     __yaql_format_function, __message, args, kwargs)
 
+    @specs.parameter('_Logger__message', yaqltypes.String())
     @inject_format
     def warning(__self, __yaql_format_function, __message, *args, **kwargs):
         __self._log(__self._underlying_logger.warning,
                     __yaql_format_function, __message, args, kwargs)
 
+    @specs.parameter('_Logger__message', yaqltypes.String())
     @inject_format
     def error(__self, __yaql_format_function, __message, *args, **kwargs):
         __self._log(__self._underlying_logger.error,
                     __yaql_format_function, __message, args, kwargs)
 
+    @specs.parameter('_Logger__message', yaqltypes.String())
     @inject_format
     def critical(__self, __yaql_format_function,
                  __message, *args, **kwargs):
         __self._log(__self._underlying_logger.critical,
                     __yaql_format_function, __message, args, kwargs)
 
+    @specs.parameter('_Logger__message', yaqltypes.String())
     @inject_format
     def exception(__self, __yaql_format_function,
                   __exc, __message, *args, **kwargs):
         """Print error message and stacktrace"""
-        stack_trace_message = '\n'.join([
+        stack_trace_message = u'\n'.join([
             __self._format_without_exceptions(
                 __yaql_format_function, __message, args, kwargs),
             __exc['stackTrace']().toString()
@@ -95,7 +102,7 @@ class Logger(object):
             # NOTE(akhivin): we do not want break program workflow
             # even formatting parameters are incorrect
             self._underlying_logger.warning(
-                'Can not format string: {0}'.format(message))
+                u'Can not format string: {0}'.format(message))
         return message
 
     def _log(self, log_function, yaql_format_function, message, args, kwargs):

murano/tests/unit/dsl/meta/TestLogger.yaml

@@ -15,6 +15,7 @@ Methods:
           Contract: $.class(sys:Logger).notNull()
     Body:
       - $log.debug('str')
+      - $log.debug('тест')
       - $log.debug('str', 1)
       - $log.debug('str {0}', message)
       - $log.debug('str {message}', message=>message)
@@ -26,6 +27,7 @@ Methods:
           Contract: $.class(sys:Logger).notNull()
     Body:
       - $log.trace('str')
+      - $log.trace('тест')
       - $log.trace('str', 1)
       - $log.trace('str {0}', message)
       - $log.trace('str {message}', message=>message)
@@ -37,6 +39,7 @@ Methods:
           Contract: $.class(sys:Logger).notNull()
     Body:
       - $log.info('str')
+      - $log.info('тест')
       - $log.info('str', 1)
       - $log.info('str {0}', message)
       - $log.info('str {message}', message=>message)
@@ -48,6 +51,7 @@ Methods:
           Contract: $.class(sys:Logger).notNull()
     Body:
       - $log.warning('str')
+      - $log.warning('тест')
       - $log.warning('str', 1)
       - $log.warning('str {0}', message)
       - $log.warning('str {message}', message=>message)
@@ -59,6 +63,7 @@ Methods:
           Contract: $.class(sys:Logger).notNull()
     Body:
       - $log.error('str')
+      - $log.error('тест')
       - $log.error('str', 1)
       - $log.error('str {0}', message)
       - $log.error('str {message}', message=>message)
@@ -70,6 +75,7 @@ Methods:
           Contract: $.class(sys:Logger).notNull()
     Body:
       - $log.critical('str')
+      - $log.critical('тест')
       - $log.critical('str', 1)
       - $log.critical('str {0}', message)
       - $log.critical('str {message}', message=>message)
@@ -87,6 +93,7 @@ Methods:
         As: e
         Do:
           - $log.exception($e, 'str')
+          - $log.exception($e, 'тест')
           - $log.exception($e, 'str', 1)
           - $log.exception($e, 'str {0}', message)
           - $log.exception($e, 'str {message}', message=>message)
@@ -98,4 +105,3 @@ Methods:
     Body:
       - Throw: exceptionName
         Message: exception message
-

murano/tests/unit/dsl/meta/TestUnicode.yaml

@@ -0,0 +1,29 @@
+Name: TestUnicode
+
+Methods:
+  testLiteral:
+    Body:
+      Return: солнце ♥ φεγγάρι
+
+  testExpression:
+    Body:
+      - Return:  ('солнце ♥' + ' φεγγάρι').toUpper()
+
+
+  testParameter:
+    Body:
+      - Return:  $.foo('солнце ♥ φεγγάρι')
+
+
+  testException:
+    Body:
+      - Throw: Exception
+        Message: солнце ♥ φεγγάρι
+
+
+  foo:
+    Arguments:
+      arg:
+        Contract: $.string().notNull()
+    Body:
+      Return: $arg.toUpper()

murano/tests/unit/dsl/test_logger.py

@@ -1,3 +1,4 @@
+# coding: utf-8
 # Copyright (c) 2015 Mirantis, Inc.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -26,6 +27,7 @@ class TestLogger(test_case.DslTestCase):
 
     FORMAT_CALLS = [
         call(ANY, 'str', (), {}),
+        call(ANY, u'тест', (), {}),
         call(ANY, 'str', (1,), {}),
         call(ANY, 'str {0}', ('message',), {}),
         call(ANY, 'str {message}', (), {'message': 'message'}),

murano/tests/unit/dsl/test_unicode.py

@@ -0,0 +1,45 @@
+# coding: utf-8
+# Copyright (c) 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from murano.dsl import dsl_exception
+from murano.tests.unit.dsl.foundation import object_model as om
+from murano.tests.unit.dsl.foundation import test_case
+
+
+class TestUnicode(test_case.DslTestCase):
+    def setUp(self):
+        super(TestUnicode, self).setUp()
+        self._runner = self.new_runner(om.Object('TestUnicode'))
+
+    def test_literal(self):
+        self.assertEqual(
+            u"солнце ♥ φεγγάρι",
+            self._runner.testLiteral())
+
+    def test_expression(self):
+        self.assertEqual(
+            u"СОЛНЦЕ ♥ ΦΕΓΓΆΡΙ",
+            self._runner.testExpression())
+
+    def test_parameter(self):
+        self.assertEqual(
+            u"СОЛНЦЕ ♥ ΦΕΓΓΆΡΙ",
+            self._runner.testParameter())
+
+    def test_exception(self):
+        x = self.assertRaises(
+            dsl_exception.MuranoPlException,
+            self._runner.testException)
+        self.assertEqual(u"солнце ♥ φεγγάρι", x.message)

murano/tests/unit/test_engine.py

@@ -149,21 +149,6 @@ class TestYaqlExpression(base.MuranoTestCase):
 
         self.assertEqual('string', yaql_expr.expression)
 
-    def test_unicode_expression(self):
-        yaql_expr = yaql_expression.YaqlExpression(u"'yaql ♥ unicode'",
-                                                   self._version)
-
-        self.assertEqual(u"'yaql ♥ unicode'".encode('utf-8'),
-                         yaql_expr.expression)
-
-    def test_unicode_expression_expression(self):
-        yaql_expr = yaql_expression.YaqlExpression(u"'yaql ♥ unicode'",
-                                                   self._version)
-        yaql_expr2 = yaql_expression.YaqlExpression(yaql_expr, self._version)
-
-        self.assertEqual(u"'yaql ♥ unicode'".encode('utf-8'),
-                         yaql_expr2.expression)
-
     def test_evaluate_calls(self):
         string = 'string'
         expected_calls = [mock.call(string, self._version),