From 9e8cec8910f88c8cec5fc448a7b152143d692303 Mon Sep 17 00:00:00 2001 From: Stan Lagun Date: Fri, 11 Sep 2015 02:39:20 +0300 Subject: [PATCH] Support for Unicode strings in MuranoPL was fixed There were several issues with Unicode support in MuranoPL after switch to yaql 1.0. Each of those issues caused any deploy to fail if it used to do anything with non-ASCII strings Fixed issues are: * Unicode strings should not be encoded to str types anymore. yaql 1.0 has native support for the Unicode and fails when non- ASCII chars encounter in str expressions. Also tests for Unicode support are now part of the yaql * Traces of execution were logged not as Unicode strings. Because those traces contain parameter values ant function return value logging failed when any of above contained non-ASCII chars * Stack trace logging failed when frame expression contained non-ASCII chars * Exception messages could not contain non-ASCII chars Also Logging API was not Unicode ready Change-Id: Ief0b45f15669c5f8ee74fd6ff41fa5bc39c9500b Closes-Bug: #1494275 --- murano/dsl/dsl_exception.py | 2 +- murano/dsl/executor.py | 20 ++++----- murano/dsl/principal_objects/stack_trace.py | 8 ++-- murano/dsl/virtual_exceptions.py | 6 +-- murano/dsl/yaql_expression.py | 3 +- murano/engine/system/logger.py | 13 ++++-- murano/tests/unit/dsl/meta/TestLogger.yaml | 8 +++- murano/tests/unit/dsl/meta/TestUnicode.yaml | 29 +++++++++++++ murano/tests/unit/dsl/test_logger.py | 2 + murano/tests/unit/dsl/test_unicode.py | 45 +++++++++++++++++++++ murano/tests/unit/test_engine.py | 15 ------- 11 files changed, 113 insertions(+), 38 deletions(-) create mode 100644 murano/tests/unit/dsl/meta/TestUnicode.yaml create mode 100644 murano/tests/unit/dsl/test_unicode.py diff --git a/murano/dsl/dsl_exception.py b/murano/dsl/dsl_exception.py index 2262b668..90e17d82 100644 --- a/murano/dsl/dsl_exception.py +++ b/murano/dsl/dsl_exception.py @@ -20,7 +20,7 @@ from murano.dsl.principal_objects import stack_trace class MuranoPlException(Exception): def __init__(self, names, message, stacktrace, extra=None, cause=None): super(MuranoPlException, self).__init__( - '[{0}]: {1}'.format(', '.join(names), message)) + u'[{0}]: {1}'.format(', '.join(names), message)) if not isinstance(names, list): names = [names] self._names = names diff --git a/murano/dsl/executor.py b/murano/dsl/executor.py index 74bcd35a..13f5e582 100644 --- a/murano/dsl/executor.py +++ b/murano/dsl/executor.py @@ -145,10 +145,10 @@ class MuranoDslExecutor(object): def _log_method(self, context, args, kwargs): method = helpers.get_current_method(context) param_gen = itertools.chain( - (str(arg) for arg in args), - ('{0} => {1}'.format(name, value) + (unicode(arg) for arg in args), + (u'{0} => {1}'.format(name, value) for name, value in kwargs.iteritems())) - params_str = ', '.join(param_gen) + params_str = u', '.join(param_gen) method_name = '{0}::{1}'.format(method.murano_class.name, method.name) thread_id = helpers.get_current_thread_id() caller_str = '' @@ -158,18 +158,20 @@ class MuranoDslExecutor(object): if frame['location']: caller_str = ' called from ' + stack_trace.format_frame(frame) - LOG.trace('{thread}: Begin execution {method}({params}){caller}' + LOG.trace(u'{thread}: Begin execution {method}({params}){caller}' .format(thread=thread_id, method=method_name, params=params_str, caller=caller_str)) try: def log_result(result): - LOG.trace('{thread}: End execution {method} with result ' - '{result}'.format(thread=thread_id, - method=method_name, result=result)) + LOG.trace( + u'{thread}: End execution {method} with result ' + u'{result}'.format( + thread=thread_id, method=method_name, result=result)) yield log_result except Exception as e: - LOG.trace('{thread}: End execution {method} with exception {exc}' - .format(thread=thread_id, method=method_name, exc=e)) + LOG.trace( + u'{thread}: End execution {method} with exception ' + u'{exc}'.format(thread=thread_id, method=method_name, exc=e)) raise @staticmethod diff --git a/murano/dsl/principal_objects/stack_trace.py b/murano/dsl/principal_objects/stack_trace.py index 5d12947f..c13bd2ee 100644 --- a/murano/dsl/principal_objects/stack_trace.py +++ b/murano/dsl/principal_objects/stack_trace.py @@ -66,7 +66,7 @@ def compose_stack_frame(context): method = helpers.get_current_method(context) return { 'instruction': None if instruction is None - else str(instruction), + else unicode(instruction), 'location': None if instruction is None else instruction.source_file_position, @@ -94,10 +94,10 @@ def format_frame(frame, prefix=''): instruction, prefix ) - return ('{5}File "{0}", line {1}{2} in method {3}\n' - '{5} {4}').format(*args) + return (u'{5}File "{0}", line {1}{2} in method {3}\n' + u'{5} {4}').format(*args) else: - return '{2}File in method {0}\n{2} {1}'.format( + return u'{2}File in method {0}\n{2} {1}'.format( method, instruction, prefix) diff --git a/murano/dsl/virtual_exceptions.py b/murano/dsl/virtual_exceptions.py index a2323cb4..dce9a83d 100644 --- a/murano/dsl/virtual_exceptions.py +++ b/murano/dsl/virtual_exceptions.py @@ -50,10 +50,10 @@ class ThrowMacro(expressions.DslExpression): helpers.evaluate(self._message, context), stacktrace, self._extra, cause) - def __str__(self): + def __unicode__(self): if self._message: - return 'Throw {0}: {1}'.format(self._names, self._message) - return 'Throw ' + str(self._names) + return u'Throw {0}: {1}'.format(self._names, self._message) + return u'Throw ' + unicode(self._names) class CatchBlock(expressions.DslExpression): diff --git a/murano/dsl/yaql_expression.py b/murano/dsl/yaql_expression.py index 326c547d..e6ea0d3f 100644 --- a/murano/dsl/yaql_expression.py +++ b/murano/dsl/yaql_expression.py @@ -15,7 +15,6 @@ import re import types -from oslo_utils import encodeutils from yaql.language import exceptions as yaql_exceptions from yaql.language import expressions @@ -28,7 +27,7 @@ class YaqlExpression(dsl_types.YaqlExpression): def __init__(self, expression, version): self._version = version if isinstance(expression, types.StringTypes): - self._expression = encodeutils.safe_encode(expression) + self._expression = unicode(expression) self._parsed_expression = yaql_integration.parse( self._expression, version) self._file_position = None diff --git a/murano/engine/system/logger.py b/murano/engine/system/logger.py index bba8bb42..4a4a1dbd 100644 --- a/murano/engine/system/logger.py +++ b/murano/engine/system/logger.py @@ -21,7 +21,7 @@ from yaql.language import yaqltypes from murano.dsl import dsl -NAME_TEMPLATE = 'applications.{0}' +NAME_TEMPLATE = u'applications.{0}' inject_format = specs.inject( '_Logger__yaql_format_function', @@ -39,42 +39,49 @@ class Logger(object): self._underlying_logger = logging.getLogger( NAME_TEMPLATE.format(logger_name)) + @specs.parameter('_Logger__message', yaqltypes.String()) @inject_format def trace(__self, __yaql_format_function, __message, *args, **kwargs): __self._log(__self._underlying_logger.trace, __yaql_format_function, __message, args, kwargs) + @specs.parameter('_Logger__message', yaqltypes.String()) @inject_format def debug(__self, __yaql_format_function, __message, *args, **kwargs): __self._log(__self._underlying_logger.debug, __yaql_format_function, __message, args, kwargs) + @specs.parameter('_Logger__message', yaqltypes.String()) @inject_format def info(__self, __yaql_format_function, __message, *args, **kwargs): __self._log(__self._underlying_logger.info, __yaql_format_function, __message, args, kwargs) + @specs.parameter('_Logger__message', yaqltypes.String()) @inject_format def warning(__self, __yaql_format_function, __message, *args, **kwargs): __self._log(__self._underlying_logger.warning, __yaql_format_function, __message, args, kwargs) + @specs.parameter('_Logger__message', yaqltypes.String()) @inject_format def error(__self, __yaql_format_function, __message, *args, **kwargs): __self._log(__self._underlying_logger.error, __yaql_format_function, __message, args, kwargs) + @specs.parameter('_Logger__message', yaqltypes.String()) @inject_format def critical(__self, __yaql_format_function, __message, *args, **kwargs): __self._log(__self._underlying_logger.critical, __yaql_format_function, __message, args, kwargs) + @specs.parameter('_Logger__message', yaqltypes.String()) @inject_format def exception(__self, __yaql_format_function, __exc, __message, *args, **kwargs): """Print error message and stacktrace""" - stack_trace_message = '\n'.join([ + stack_trace_message = u'\n'.join([ __self._format_without_exceptions( __yaql_format_function, __message, args, kwargs), __exc['stackTrace']().toString() @@ -95,7 +102,7 @@ class Logger(object): # NOTE(akhivin): we do not want break program workflow # even formatting parameters are incorrect self._underlying_logger.warning( - 'Can not format string: {0}'.format(message)) + u'Can not format string: {0}'.format(message)) return message def _log(self, log_function, yaql_format_function, message, args, kwargs): diff --git a/murano/tests/unit/dsl/meta/TestLogger.yaml b/murano/tests/unit/dsl/meta/TestLogger.yaml index f351cc62..c1f90c2b 100644 --- a/murano/tests/unit/dsl/meta/TestLogger.yaml +++ b/murano/tests/unit/dsl/meta/TestLogger.yaml @@ -15,6 +15,7 @@ Methods: Contract: $.class(sys:Logger).notNull() Body: - $log.debug('str') + - $log.debug('тест') - $log.debug('str', 1) - $log.debug('str {0}', message) - $log.debug('str {message}', message=>message) @@ -26,6 +27,7 @@ Methods: Contract: $.class(sys:Logger).notNull() Body: - $log.trace('str') + - $log.trace('тест') - $log.trace('str', 1) - $log.trace('str {0}', message) - $log.trace('str {message}', message=>message) @@ -37,6 +39,7 @@ Methods: Contract: $.class(sys:Logger).notNull() Body: - $log.info('str') + - $log.info('тест') - $log.info('str', 1) - $log.info('str {0}', message) - $log.info('str {message}', message=>message) @@ -48,6 +51,7 @@ Methods: Contract: $.class(sys:Logger).notNull() Body: - $log.warning('str') + - $log.warning('тест') - $log.warning('str', 1) - $log.warning('str {0}', message) - $log.warning('str {message}', message=>message) @@ -59,6 +63,7 @@ Methods: Contract: $.class(sys:Logger).notNull() Body: - $log.error('str') + - $log.error('тест') - $log.error('str', 1) - $log.error('str {0}', message) - $log.error('str {message}', message=>message) @@ -70,6 +75,7 @@ Methods: Contract: $.class(sys:Logger).notNull() Body: - $log.critical('str') + - $log.critical('тест') - $log.critical('str', 1) - $log.critical('str {0}', message) - $log.critical('str {message}', message=>message) @@ -87,6 +93,7 @@ Methods: As: e Do: - $log.exception($e, 'str') + - $log.exception($e, 'тест') - $log.exception($e, 'str', 1) - $log.exception($e, 'str {0}', message) - $log.exception($e, 'str {message}', message=>message) @@ -98,4 +105,3 @@ Methods: Body: - Throw: exceptionName Message: exception message - diff --git a/murano/tests/unit/dsl/meta/TestUnicode.yaml b/murano/tests/unit/dsl/meta/TestUnicode.yaml new file mode 100644 index 00000000..f6dc3eb0 --- /dev/null +++ b/murano/tests/unit/dsl/meta/TestUnicode.yaml @@ -0,0 +1,29 @@ +Name: TestUnicode + +Methods: + testLiteral: + Body: + Return: солнце ♥ φεγγάρι + + testExpression: + Body: + - Return: ('солнце ♥' + ' φεγγάρι').toUpper() + + + testParameter: + Body: + - Return: $.foo('солнце ♥ φεγγάρι') + + + testException: + Body: + - Throw: Exception + Message: солнце ♥ φεγγάρι + + + foo: + Arguments: + arg: + Contract: $.string().notNull() + Body: + Return: $arg.toUpper() \ No newline at end of file diff --git a/murano/tests/unit/dsl/test_logger.py b/murano/tests/unit/dsl/test_logger.py index adeed9f8..8fb34ac3 100644 --- a/murano/tests/unit/dsl/test_logger.py +++ b/murano/tests/unit/dsl/test_logger.py @@ -1,3 +1,4 @@ +# coding: utf-8 # Copyright (c) 2015 Mirantis, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -26,6 +27,7 @@ class TestLogger(test_case.DslTestCase): FORMAT_CALLS = [ call(ANY, 'str', (), {}), + call(ANY, u'тест', (), {}), call(ANY, 'str', (1,), {}), call(ANY, 'str {0}', ('message',), {}), call(ANY, 'str {message}', (), {'message': 'message'}), diff --git a/murano/tests/unit/dsl/test_unicode.py b/murano/tests/unit/dsl/test_unicode.py new file mode 100644 index 00000000..1c8f964d --- /dev/null +++ b/murano/tests/unit/dsl/test_unicode.py @@ -0,0 +1,45 @@ +# coding: utf-8 +# Copyright (c) 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from murano.dsl import dsl_exception +from murano.tests.unit.dsl.foundation import object_model as om +from murano.tests.unit.dsl.foundation import test_case + + +class TestUnicode(test_case.DslTestCase): + def setUp(self): + super(TestUnicode, self).setUp() + self._runner = self.new_runner(om.Object('TestUnicode')) + + def test_literal(self): + self.assertEqual( + u"солнце ♥ φεγγάρι", + self._runner.testLiteral()) + + def test_expression(self): + self.assertEqual( + u"СОЛНЦЕ ♥ ΦΕΓΓΆΡΙ", + self._runner.testExpression()) + + def test_parameter(self): + self.assertEqual( + u"СОЛНЦЕ ♥ ΦΕΓΓΆΡΙ", + self._runner.testParameter()) + + def test_exception(self): + x = self.assertRaises( + dsl_exception.MuranoPlException, + self._runner.testException) + self.assertEqual(u"солнце ♥ φεγγάρι", x.message) diff --git a/murano/tests/unit/test_engine.py b/murano/tests/unit/test_engine.py index a40e758b..c9d34655 100644 --- a/murano/tests/unit/test_engine.py +++ b/murano/tests/unit/test_engine.py @@ -149,21 +149,6 @@ class TestYaqlExpression(base.MuranoTestCase): self.assertEqual('string', yaql_expr.expression) - def test_unicode_expression(self): - yaql_expr = yaql_expression.YaqlExpression(u"'yaql ♥ unicode'", - self._version) - - self.assertEqual(u"'yaql ♥ unicode'".encode('utf-8'), - yaql_expr.expression) - - def test_unicode_expression_expression(self): - yaql_expr = yaql_expression.YaqlExpression(u"'yaql ♥ unicode'", - self._version) - yaql_expr2 = yaql_expression.YaqlExpression(yaql_expr, self._version) - - self.assertEqual(u"'yaql ♥ unicode'".encode('utf-8'), - yaql_expr2.expression) - def test_evaluate_calls(self): string = 'string' expected_calls = [mock.call(string, self._version),