diff --git a/doc/source/template_guide/hot_spec.rst b/doc/source/template_guide/hot_spec.rst index 335a006da..113dc3de6 100644 --- a/doc/source/template_guide/hot_spec.rst +++ b/doc/source/template_guide/hot_spec.rst @@ -203,6 +203,27 @@ For example, Heat currently supports the following values for the str_replace str_split +2016-10-14 +---------- + The key with value ``2016-10-14`` indicates that the YAML document is a HOT + template and it may contain features added and/or removed up until the + Newton release. This version also adds the yaql function which + can be used for evaluation of complex expressions. The complete list of + supported functions is:: + + digest + get_attr + get_file + get_param + get_resource + list_join + map_merge + repeat + resource_facade + str_replace + str_split + yaql + .. _hot_spec_parameter_groups: Parameter groups section @@ -1264,3 +1285,33 @@ For example This resolves to a map containing ``{'k1': 'v2', 'k2': 'v2'}``. Maps containing no items resolve to {}. + +yaql +---- +The ``yaql`` evaluates yaql expression on a given data. + +The syntax of the ``yaql`` function is + +.. code-block:: yaml + + yaql: + expression: + data: + +For example + +.. code-block:: yaml + + parameters: + list_param: + type: comma_delimited_list + default: [1, 2, 3] + + outputs: + max_elem: + yaql: + expression: $.data.list_param.select(int($)).max() + data: + list_param: {get_param: list_param} + +max_elem output will be evaluated to 3 diff --git a/heat/engine/hot/functions.py b/heat/engine/hot/functions.py index 1e0a23be3..4b295b025 100644 --- a/heat/engine/hot/functions.py +++ b/heat/engine/hot/functions.py @@ -14,9 +14,12 @@ import collections import hashlib import itertools -import six +from oslo_config import cfg from oslo_serialization import jsonutils +import six +import yaql +from yaql.language import exceptions from heat.common import exception from heat.common.i18n import _ @@ -24,6 +27,18 @@ from heat.engine import attributes from heat.engine.cfn import functions as cfn_funcs from heat.engine import function +opts = [ + cfg.IntOpt('limit_iterators', + default=200, + help=_('The maximum number of elements in collection ' + 'expression can take for its evaluation.')), + cfg.IntOpt('memory_quota', + default=10000, + help=_('The maximum size of memory in bytes that ' + 'expression can take for its evaluation.')) +] +cfg.CONF.register_opts(opts, group='yaql') + class GetParam(function.Function): """A function for resolving parameter references. @@ -697,3 +712,85 @@ class StrSplit(function.Function): else: res = split_list return res + + +class Yaql(function.Function): + """A function for executing a yaql expression. + + Takes the form:: + + yaql: + expression: + + data: + : + + Evaluates expression on the given data. + """ + + _parser = None + + @classmethod + def get_yaql_parser(cls): + if cls._parser is None: + global_options = { + 'yaql.limitIterators': cfg.CONF.yaql.limit_iterators, + 'yaql.memoryQuota': cfg.CONF.yaql.memory_quota + } + cls._parser = yaql.YaqlFactory().create(global_options) + return cls._parser + + def __init__(self, stack, fn_name, args): + super(Yaql, self).__init__(stack, fn_name, args) + self.parser = self.get_yaql_parser() + self.context = yaql.create_context() + + if not isinstance(self.args, collections.Mapping): + raise TypeError(_('Arguments to "%s" must be a map.') % + self.fn_name) + + try: + self._expression = self.args['expression'] + self._data = self.args.get('data', {}) + for arg in six.iterkeys(self.args): + if arg not in ['expression', 'data']: + raise KeyError + except (KeyError, TypeError): + example = ('''%s: + expression: $.data.var1.sum() + data: + var1: [3, 2, 1]''') % self.fn_name + raise KeyError(_('"%(name)s" syntax should be %(example)s') % { + 'name': self.fn_name, 'example': example}) + + def validate_expression(self, expression): + try: + self.parser(expression) + except exceptions.YaqlException as yex: + raise ValueError(_('Bad expression %s.') % yex) + + def validate(self): + super(Yaql, self).validate() + if not isinstance(self._data, + (collections.Mapping, function.Function)): + raise TypeError(_('The "data" argument to "%s" must contain ' + 'a map.') % self.fn_name) + if not isinstance(self._expression, + (six.string_types, function.Function)): + raise TypeError(_('The "expression" argument to %s must ' + 'contain a string or a ' + 'function.') % self.fn_name) + if isinstance(self._expression, six.string_types): + self.validate_expression(self._expression) + + def result(self): + data = function.resolve(self._data) + if not isinstance(data, collections.Mapping): + raise TypeError(_('The "data" argument to "%s" must contain ' + 'a map.') % self.fn_name) + ctxt = {'data': data} + self.context['$'] = ctxt + if not isinstance(self._expression, six.string_types): + self._expression = function.resolve(self._expression) + self.validate_expression(self._expression) + return self.parser(self._expression).evaluate(context=self.context) diff --git a/heat/engine/hot/template.py b/heat/engine/hot/template.py index 784382294..f9a952f36 100644 --- a/heat/engine/hot/template.py +++ b/heat/engine/hot/template.py @@ -411,6 +411,9 @@ class HOTemplate20161014(HOTemplate20160408): 'resource_facade': hot_funcs.ResourceFacade, 'str_replace': hot_funcs.ReplaceJson, + # functions added since 20161014 + 'yaql': hot_funcs.Yaql, + # functions added since 20151015 'map_merge': hot_funcs.MapMerge, diff --git a/heat/tests/test_hot.py b/heat/tests/test_hot.py index c3ad073c9..7004dca17 100644 --- a/heat/tests/test_hot.py +++ b/heat/tests/test_hot.py @@ -57,6 +57,10 @@ hot_mitaka_tpl_empty = template_format.parse(''' heat_template_version: 2016-04-08 ''') +hot_newton_tpl_empty = template_format.parse(''' +heat_template_version: 2016-10-14 +''') + hot_tpl_empty_sections = template_format.parse(''' heat_template_version: 2013-05-23 parameters: @@ -833,6 +837,60 @@ class HOTemplateTest(common.HeatTestCase): self.assertEqual('role1', resolved['role1']) self.assertEqual('role2', resolved['role2']) + def test_yaql(self): + snippet = {'yaql': {'expression': '$.data.var1.sum()', + 'data': {'var1': [1, 2, 3, 4]}}} + tmpl = template.Template(hot_newton_tpl_empty) + stack = parser.Stack(utils.dummy_context(), 'test_stack', tmpl) + resolved = self.resolve(snippet, tmpl, stack=stack) + + self.assertEqual(10, resolved) + + def test_yaql_invalid_data(self): + snippet = {'yaql': {'expression': '$.data.var1.sum()', + 'data': 'mustbeamap'}} + tmpl = template.Template(hot_newton_tpl_empty) + msg = 'The "data" argument to "yaql" must contain a map.' + self.assertRaisesRegexp(TypeError, msg, self.resolve, snippet, tmpl) + + def test_yaql_bogus_keys(self): + snippet = {'yaql': {'expression': '1 + 3', + 'data': 'mustbeamap', + 'bogus': ""}} + tmpl = template.Template(hot_newton_tpl_empty) + self.assertRaises(KeyError, self.resolve, snippet, tmpl) + + def test_yaql_invalid_syntax(self): + snippet = {'yaql': {'wrong': 'wrong_expr', + 'wrong_data': 'mustbeamap'}} + tmpl = template.Template(hot_newton_tpl_empty) + self.assertRaises(KeyError, self.resolve, snippet, tmpl) + + def test_yaql_non_map_args(self): + snippet = {'yaql': 'invalid'} + tmpl = template.Template(hot_newton_tpl_empty) + msg = 'Arguments to "yaql" must be a map.' + self.assertRaisesRegexp(TypeError, msg, self.resolve, snippet, tmpl) + + def test_yaql_invalid_expression(self): + snippet = {'yaql': {'expression': 'invalid(', + 'data': {'var1': [1, 2, 3, 4]}}} + tmpl = template.Template(hot_newton_tpl_empty) + yaql = tmpl.parse(None, snippet) + self.assertRaises(ValueError, function.validate, yaql) + + def test_yaql_data_as_function(self): + snippet = {'yaql': {'expression': '$.data.var1.len()', + 'data': { + 'var1': {'list_join': ['', ['1', '2']]} + } + }} + tmpl = template.Template(hot_newton_tpl_empty) + stack = parser.Stack(utils.dummy_context(), 'test_stack', tmpl) + resolved = self.resolve(snippet, tmpl, stack=stack) + + self.assertEqual(2, resolved) + def test_repeat(self): """Test repeat function.""" snippet = {'repeat': {'template': 'this is %var%', diff --git a/requirements.txt b/requirements.txt index ed85cc520..75d980436 100644 --- a/requirements.txt +++ b/requirements.txt @@ -60,3 +60,4 @@ SQLAlchemy<1.1.0,>=1.0.10 # MIT sqlalchemy-migrate>=0.9.6 # Apache-2.0 stevedore>=1.10.0 # Apache-2.0 WebOb>=1.2.3 # MIT +yaql>=1.1.0 # Apache 2.0 License