openstack/heat
Code Issues Proposed changes

Support 'permutations' for 'repeat' function

Browse Source 
Adds a new section 'permutations' for 'repeat'
function to decide whether to do nested loops
to iterate over all the permutations of
the elements in the given lists, the default value
is True if no specify and keep the same behavior as
before.
For example:

  repeat:
    template:
      network: %net%
      port: %port%
      ip: %ip%
    for_each:
      %net%: [n1, n2]
      %port%: [p1, p2]
      %ip%: [ip1, ip2]
    permutations: False

Will be resolve to:
   [{network: n1, port: p1, ip: ip1},
    {network: n2, port: p2, ip: ip2}]

Change-Id: I2a008400fb71453f6a78656f2e041ae2efa098a2
Blueprint: improve-repeat-function
This commit is contained in:
huangtianhua 2017-06-12 18:42:16 +08:00
parent 3ae539bbf9
commit ba7f7888f6
4 changed files with 210 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
doc/source/template_guide/hot_spec.rst
View File

@ -1410,6 +1410,44 @@ nested for-loops in most programming languages.
From HOT version ``2016-10-14`` you may also pass a map as value for the
``for_each`` key, in which case the list of map keys will be used as value.
From HOT version ``2017-09-01`` (or pike) you may specify a argument
``permutations`` to decide whether to iterate nested the over all the
permutations of the elements in the given lists. If 'permutations' is not
specified, we set the default value to true to compatible with before behavior.
The args have to be lists instead of dicts if 'permutations' is False because
keys in a dict are unordered, and the list args all have to be of the
same length.
.. code-block:: yaml
parameters:
subnets:
type: comma_delimited_list
label: subnets
default: "sub1, sub2"
networks:
type: comma_delimited_list
label: networks
default: "net1, net2"
resources:
my_server:
type: OS::Nova:Server
properties:
networks:
repeat:
for_each:
<%sub%>: { get_param: subnets }
<%net%>: { get_param: networks }
template:
subnet: <%sub%>
network: <%net%>
permutations: false
After resolved, we will get the networks of server like:
[{subnet: sub1, network: net1}, {subnet: sub2, network: net2}]
resource_facade
---------------
The ``resource_facade`` function retrieves data in a parent

81
heat/engine/hot/functions.py
View File

@ -887,9 +887,12 @@ class Repeat(function.Function):
is a copy of <body> with any occurrences of <var> replaced with the
corresponding item of <list>.
"""
def __init__(self, stack, fn_name, args):
super(Repeat, self).__init__(stack, fn_name, args)
self._parse_args()
def _parse_args(self):
if not isinstance(self.args, collections.Mapping):
raise TypeError(_('Arguments to "%s" must be a map') %
self.fn_name)
@ -906,6 +909,8 @@ class Repeat(function.Function):
%var%: ['a', 'b', 'c']''')
raise KeyError(_('"repeat" syntax should be %s') % example)
self._nested_loop = True
def validate(self):
super(Repeat, self).validate()
@ -942,20 +947,37 @@ class Repeat(function.Function):
# use empty list for references(None) else validation will fail
values = [[] if value is None else value for value in lists]
value_lens = []
for arg in values:
self._valid_arg(arg)
value_lens.append(len(arg))
if not self._nested_loop:
if len(set(value_lens)) != 1:
raise ValueError(_('For %s, the length of for_each values '
'should be equal if no nested '
'loop.') % self.fn_name)
template = function.resolve(self._template)
iter_func = itertools.product if self._nested_loop else six.moves.zip
return [self._do_replacement(keys, replacements, template)
for replacements in itertools.product(*values)]
for replacements in iter_func(*values)]
class RepeatWithMap(Repeat):
"""A function for iterating over a list or map of items.
"""A function for iterating over a list of items or a dict of keys.
Behaves the same as Repeat, but if tolerates a map as
values to be repeated, in which case it iterates the map keys.
Takes the form::
repeat:
template:
<body>
for_each:
<var>: <list> or <dict>
The result is a new list of the same size as <list> or <dict>, where each
element is a copy of <body> with any occurrences of <var> replaced with the
corresponding item of <list> or key of <dict>.
"""
def _valid_arg(self, arg):
@ -967,6 +989,57 @@ class RepeatWithMap(Repeat):
'"%s" must be lists or maps') % self.fn_name)
class RepeatWithNestedLoop(RepeatWithMap):
"""A function for iterating over a list of items or a dict of keys.
Takes the form::
repeat:
template:
<body>
for_each:
<var>: <list> or <dict>
The result is a new list of the same size as <list> or <dict>, where each
element is a copy of <body> with any occurrences of <var> replaced with the
corresponding item of <list> or key of <dict>.
This function also allows to specify 'permutations' to decide
whether to iterate nested the over all the permutations of the
elements in the given lists.
Takes the form::
repeat:
template:
var: %var%
bar: %bar%
for_each:
%var%: <list1>
%bar%: <list2>
permutations: false
If 'permutations' is not specified, we set the default value to true to
compatible with before behavior. The args have to be lists instead of
dicts if 'permutations' is False because keys in a dict are unordered,
and the list args all have to be of the same length.
"""
def _parse_args(self):
super(RepeatWithNestedLoop, self)._parse_args()
self._nested_loop = self.args.get('permutations', True)
if not isinstance(self._nested_loop, bool):
raise TypeError(_('"permutations" should be boolean type '
'for %s function.') % self.fn_name)
def _valid_arg(self, arg):
if self._nested_loop:
super(RepeatWithNestedLoop, self)._valid_arg(arg)
else:
Repeat._valid_arg(self, arg)
class Digest(function.Function):
"""A function for performing digest operations.

2
heat/engine/hot/template.py
View File

@ -571,7 +571,7 @@ class HOTemplate20170901(HOTemplate20170224):
'get_param': hot_funcs.GetParam,
'get_resource': hot_funcs.GetResource,
'list_join': hot_funcs.JoinMultiple,
'repeat': hot_funcs.RepeatWithMap,
'repeat': hot_funcs.RepeatWithNestedLoop,
'resource_facade': hot_funcs.ResourceFacade,
'str_replace': hot_funcs.ReplaceJson,

104
heat/tests/test_hot.py
View File

@ -1448,16 +1448,22 @@ conditions:
six.text_type(exc))
self.assertIn('if:', six.text_type(exc))
def test_repeat(self):
def _test_repeat(self, templ=hot_kilo_tpl_empty):
"""Test repeat function."""
snippet = {'repeat': {'template': 'this is %var%',
'for_each': {'%var%': ['a', 'b', 'c']}}}
snippet_resolved = ['this is a', 'this is b', 'this is c']
tmpl = template.Template(hot_kilo_tpl_empty)
tmpl = template.Template(templ)
self.assertEqual(snippet_resolved, self.resolve(snippet, tmpl))
def test_repeat(self):
self._test_repeat()
def test_repeat_with_pike_version(self):
self._test_repeat(templ=hot_pike_tpl_empty)
def test_repeat_get_param(self):
"""Test repeat function with get_param function as an argument."""
hot_tpl = template_format.parse('''
@ -1476,16 +1482,23 @@ conditions:
self.assertEqual(snippet_resolved, self.resolve(snippet, tmpl, stack))
def test_repeat_dict_with_no_replacement(self):
def _test_repeat_dict_with_no_replacement(self,
templ=hot_newton_tpl_empty):
snippet = {'repeat': {'template': {'SERVICE_enabled': True},
'for_each': {'SERVICE': ['x', 'y', 'z']}}}
snippet_resolved = [{'x_enabled': True},
{'y_enabled': True},
{'z_enabled': True}]
tmpl = template.Template(hot_newton_tpl_empty)
tmpl = template.Template(templ)
self.assertEqual(snippet_resolved, self.resolve(snippet, tmpl))
def test_repeat_dict_template(self):
def test_repeat_dict_with_no_replacement(self):
self._test_repeat_dict_with_no_replacement()
def test_repeat_dict_with_no_replacement_pike_version(self):
self._test_repeat_dict_with_no_replacement(templ=hot_pike_tpl_empty)
def _test_repeat_dict_template(self, templ=hot_kilo_tpl_empty):
"""Test repeat function with a dictionary as a template."""
snippet = {'repeat': {'template': {'key-%var%': 'this is %var%'},
'for_each': {'%var%': ['a', 'b', 'c']}}}
@ -1493,11 +1506,17 @@ conditions:
{'key-b': 'this is b'},
{'key-c': 'this is c'}]
tmpl = template.Template(hot_kilo_tpl_empty)
tmpl = template.Template(templ)
self.assertEqual(snippet_resolved, self.resolve(snippet, tmpl))
def test_repeat_list_template(self):
def test_repeat_dict_template(self):
self._test_repeat_dict_template()
def test_repeat_dict_template_pike_version(self):
self._test_repeat_dict_template(templ=hot_pike_tpl_empty)
def _test_repeat_list_template(self, templ=hot_kilo_tpl_empty):
"""Test repeat function with a list as a template."""
snippet = {'repeat': {'template': ['this is %var%', 'static'],
'for_each': {'%var%': ['a', 'b', 'c']}}}
@ -1505,11 +1524,17 @@ conditions:
['this is b', 'static'],
['this is c', 'static']]
tmpl = template.Template(hot_kilo_tpl_empty)
tmpl = template.Template(templ)
self.assertEqual(snippet_resolved, self.resolve(snippet, tmpl))
def test_repeat_multi_list(self):
def test_repeat_list_template(self):
self._test_repeat_list_template()
def test_repeat_list_template_pike_version(self):
self._test_repeat_list_template(templ=hot_pike_tpl_empty)
def _test_repeat_multi_list(self, templ=hot_kilo_tpl_empty):
"""Test repeat function with multiple input lists."""
snippet = {'repeat': {'template': 'this is %var1%-%var2%',
'for_each': {'%var1%': ['a', 'b', 'c'],
@ -1517,13 +1542,19 @@ conditions:
snippet_resolved = ['this is a-1', 'this is b-1', 'this is c-1',
'this is a-2', 'this is b-2', 'this is c-2']
tmpl = template.Template(hot_kilo_tpl_empty)
tmpl = template.Template(templ)
result = self.resolve(snippet, tmpl)
self.assertEqual(len(result), len(snippet_resolved))
for item in result:
self.assertIn(item, snippet_resolved)
def test_repeat_multi_list(self):
self._test_repeat_multi_list()
def test_repeat_multi_list_pike_version(self):
self._test_repeat_multi_list(templ=hot_pike_tpl_empty)
def test_repeat_list_and_map(self):
"""Test repeat function with a list and a map."""
snippet = {'repeat': {'template': 'this is %var1%-%var2%',
@ -1539,6 +1570,59 @@ conditions:
for item in result:
self.assertIn(item, snippet_resolved)
def test_repeat_with_no_nested_loop(self):
snippet = {'repeat': {'template': {'network': '%net%',
'port': '%port%',
'subnet': '%sub%'},
'for_each': {'%net%': ['n1', 'n2', 'n3', 'n4'],
'%port%': ['p1', 'p2', 'p3', 'p4'],
'%sub%': ['s1', 's2', 's3', 's4']},
'permutations': False}}
tmpl = template.Template(hot_pike_tpl_empty)
snippet_resolved = [{'network': 'n1', 'port': 'p1', 'subnet': 's1'},
{'network': 'n2', 'port': 'p2', 'subnet': 's2'},
{'network': 'n3', 'port': 'p3', 'subnet': 's3'},
{'network': 'n4', 'port': 'p4', 'subnet': 's4'}]
result = self.resolve(snippet, tmpl)
self.assertEqual(snippet_resolved, result)
def test_repeat_no_nested_loop_different_len(self):
snippet = {'repeat': {'template': {'network': '%net%',
'port': '%port%',
'subnet': '%sub%'},
'for_each': {'%net%': ['n1', 'n2', 'n3'],
'%port%': ['p1', 'p2'],
'%sub%': ['s1', 's2']},
'permutations': False}}
tmpl = template.Template(hot_pike_tpl_empty)
self.assertRaises(ValueError, self.resolve, snippet, tmpl)
def test_repeat_no_nested_loop_with_dict_type(self):
snippet = {'repeat': {'template': {'network': '%net%',
'port': '%port%',
'subnet': '%sub%'},
'for_each': {'%net%': ['n1', 'n2'],
'%port%': {'p1': 'pp', 'p2': 'qq'},
'%sub%': ['s1', 's2']},
'permutations': False}}
tmpl = template.Template(hot_pike_tpl_empty)
self.assertRaises(TypeError, self.resolve, snippet, tmpl)
def test_repeat_permutations_non_bool(self):
snippet = {'repeat': {'template': {'network': '%net%',
'port': '%port%',
'subnet': '%sub%'},
'for_each': {'%net%': ['n1', 'n2'],
'%port%': ['p1', 'p2'],
'%sub%': ['s1', 's2']},
'permutations': 'non bool'}}
tmpl = template.Template(hot_pike_tpl_empty)
exc = self.assertRaises(exception.StackValidationFailed,
self.resolve, snippet, tmpl)
self.assertIn('"permutations" should be boolean type '
'for repeat function', six.text_type(exc))
def test_repeat_bad_args(self):
"""Tests reporting error by repeat function.