Merge "Use param_schema and merge strategy for merging"
This commit is contained in:
Jenkins
Gerrit Code Review
commit
c1dec1b720
|
|
@ -12,9 +12,12 @@
|
|||
# under the License.
|
||||
import collections
|
||||
|
||||
from oslo_serialization import jsonutils
|
||||
import six
|
||||
|
||||
from heat.common import environment_format as env_fmt
|
||||
from heat.common import exception
|
||||
from heat.common.i18n import _
|
||||
|
||||
ALLOWED_PARAM_MERGE_STRATEGIES = (OVERWRITE, MERGE, DEEP_MERGE) = (
|
||||
'overwrite', 'merge', 'deep_merge')
|
||||
|
|
@ -62,14 +65,69 @@ def merge_map(old, new, deep_merge=False):
|
|||
not isinstance(v, six.string_types)):
|
||||
old_v = old.get(k)
|
||||
old[k] = merge_list(old_v, v) if old_v else v
|
||||
elif isinstance(v, six.string_types):
|
||||
old[k] = ''.join([old.get(k, ''), v])
|
||||
else:
|
||||
old[k] = v
|
||||
|
||||
return old
|
||||
|
||||
|
||||
def parse_param(p_val, p_schema):
|
||||
try:
|
||||
if p_schema.type == p_schema.MAP:
|
||||
if not isinstance(p_val, six.string_types):
|
||||
p_val = jsonutils.dumps(p_val)
|
||||
if p_val:
|
||||
return jsonutils.loads(p_val)
|
||||
elif not isinstance(p_val, collections.Sequence):
|
||||
raise ValueError()
|
||||
except (ValueError, TypeError) as err:
|
||||
msg = _("Invalid parameter in environment %(s).") % six.text_type(err)
|
||||
raise ValueError(msg)
|
||||
return p_val
|
||||
|
||||
|
||||
def merge_parameters(old, new, param_schemata,
|
||||
merge_strategies):
|
||||
|
||||
def param_merge(p_key, p_value, p_schema, deep_merge=False):
|
||||
p_type = p_schema.type
|
||||
p_value = parse_param(p_value, p_schema)
|
||||
if p_type == p_schema.MAP:
|
||||
old[p_key] = merge_map(old.get(p_key, {}), p_value, deep_merge)
|
||||
elif p_type == p_schema.LIST:
|
||||
old[p_key] = merge_list(old.get(p_key), p_value)
|
||||
elif p_type == p_schema.STRING:
|
||||
old[p_key] = ''.join([old.get(p_key, ''), p_value])
|
||||
elif p_type == p_schema.NUMBER:
|
||||
old[p_key] = old.get(p_key, 0) + p_value
|
||||
else:
|
||||
raise exception.InvalidMergeStrategyForParam(strategy=MERGE,
|
||||
param=p_key)
|
||||
|
||||
if not old:
|
||||
return new
|
||||
|
||||
for key, value in new.items():
|
||||
# if key not in param_schemata ignore it
|
||||
if key in param_schemata and value:
|
||||
param_merge_strategy = get_param_merge_strategy(merge_strategies,
|
||||
key)
|
||||
if param_merge_strategy == DEEP_MERGE:
|
||||
param_merge(key, value,
|
||||
param_schemata[key],
|
||||
deep_merge=True)
|
||||
elif param_merge_strategy == MERGE:
|
||||
param_merge(key, value, param_schemata[key])
|
||||
else:
|
||||
old[key] = value
|
||||
|
||||
return old
|
||||
|
||||
|
||||
def merge_environments(environment_files, files,
|
||||
params, param_schemata=None):
|
||||
params, param_schemata):
|
||||
"""Merges environment files into the stack input parameters.
|
||||
|
||||
If a list of environment files have been specified, this call will
|
||||
|
|
@ -85,6 +143,8 @@ def merge_environments(environment_files, files,
|
|||
:type files: dict
|
||||
:param params: parameters describing the stack
|
||||
:type dict:
|
||||
:param param_schemata: parameter schema dict
|
||||
:type param_schemata: dict
|
||||
"""
|
||||
if not environment_files:
|
||||
return
|
||||
|
|
@ -92,8 +152,17 @@ def merge_environments(environment_files, files,
|
|||
for filename in environment_files:
|
||||
raw_env = files[filename]
|
||||
parsed_env = env_fmt.parse(raw_env)
|
||||
merge_strategies = parsed_env.pop(
|
||||
env_fmt.PARAMETER_MERGE_STRATEGIES, {})
|
||||
|
||||
for section_key, section_value in parsed_env.items():
|
||||
if section_value:
|
||||
params[section_key] = merge_map(params[section_key],
|
||||
section_value,
|
||||
deep_merge=True)
|
||||
if section_key in (env_fmt.PARAMETERS,
|
||||
env_fmt.PARAMETER_DEFAULTS):
|
||||
params[section_key] = merge_parameters(params[section_key],
|
||||
section_value,
|
||||
param_schemata,
|
||||
merge_strategies)
|
||||
else:
|
||||
params[section_key] = merge_map(params[section_key],
|
||||
section_value)
|
||||
|
|
|
|||
|
|
@ -152,6 +152,11 @@ class ImmutableParameterModified(HeatException):
|
|||
super(ImmutableParameterModified, self).__init__(**kwargs)
|
||||
|
||||
|
||||
class InvalidMergeStrategyForParam(HeatException):
|
||||
msg_fmt = _("Invalid merge strategy %(strategy)s for "
|
||||
"parameter %(param)s.")
|
||||
|
||||
|
||||
class InvalidTemplateAttribute(HeatException):
|
||||
msg_fmt = _("The Referenced Attribute (%(resource)s %(key)s)"
|
||||
" is incorrect.")
|
||||
|
|
|
|||
|
|
@ -11,8 +11,11 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import json
|
||||
|
||||
from heat.common import environment_util as env_util
|
||||
from heat.tests import common
|
||||
from heat.tests import utils
|
||||
|
||||
|
||||
class TestEnvironmentUtil(common.HeatTestCase):
|
||||
|
|
@ -51,89 +54,175 @@ class TestEnvironmentUtil(common.HeatTestCase):
|
|||
|
||||
|
||||
class TestMergeEnvironments(common.HeatTestCase):
|
||||
|
||||
def test_merge_environments(self):
|
||||
def setUp(self):
|
||||
super(TestMergeEnvironments, self).setUp()
|
||||
self.ctx = utils.dummy_context(tenant_id='stack_service_test_tenant')
|
||||
# Setup
|
||||
params = {'parameters': {
|
||||
'p0': 'CORRECT',
|
||||
'p1': 'INCORRECT',
|
||||
'p2': 'INCORRECT'}
|
||||
self.params = {'parameters': {
|
||||
'str_value1': "test1",
|
||||
'str_value2': "test2",
|
||||
'del_lst_value1': '',
|
||||
'del_lst_value2': '',
|
||||
'lst_value1': [],
|
||||
'lst_value2': [],
|
||||
'json_value1': {},
|
||||
'json_value2': {}},
|
||||
'resource_registry': {}
|
||||
}
|
||||
env_1 = '''
|
||||
{'parameters' : {
|
||||
'p1': 'CORRECT',
|
||||
'p2': 'INCORRECT-ENV-1',
|
||||
}}'''
|
||||
env_2 = '''
|
||||
{'parameters': {
|
||||
'p2': 'CORRECT'
|
||||
}}'''
|
||||
|
||||
files = {'env_1': env_1, 'env_2': env_2}
|
||||
self.env_1 = {'parameters': {
|
||||
'str_value1': "string1",
|
||||
'str_value2': "string2",
|
||||
'del_lst_value1': '1,2',
|
||||
'del_lst_value2': '3,4',
|
||||
'lst_value1': [1, 2],
|
||||
'lst_value2': [3, 4],
|
||||
'json_value1': {"1": ["str1", "str2"]},
|
||||
'json_value2': {"2": ["test1", "test2"]}},
|
||||
'resource_registry': {
|
||||
'test::R1': "OS::Heat::RandomString",
|
||||
'test::R2': "BROKEN"}
|
||||
}
|
||||
self.env_2 = {'parameters': {
|
||||
'str_value1': "string3",
|
||||
'str_value2': "string4",
|
||||
'del_lst_value1': '5,6',
|
||||
'del_lst_value2': '7,8',
|
||||
'lst_value1': [5, 6],
|
||||
'lst_value2': [7, 8],
|
||||
'json_value1': {"3": ["str3", "str4"]},
|
||||
'json_value2': {"4": ["test3", "test4"]}},
|
||||
'resource_registry': {
|
||||
'test::R2': "OS::Heat::None"}
|
||||
}
|
||||
|
||||
class mock_schema(object):
|
||||
types = (MAP, LIST, STRING, NUMBER) = (
|
||||
'json', 'comma_delimited_list', 'string', 'number')
|
||||
|
||||
def __init__(self, d):
|
||||
self.__dict__ = d
|
||||
|
||||
self.param_schemata = {
|
||||
'str_value1': mock_schema({
|
||||
'type': "string"}),
|
||||
'str_value2': mock_schema({
|
||||
'type': "string"}),
|
||||
'del_lst_value1': mock_schema({
|
||||
'type': "comma_delimited_list"}),
|
||||
'del_lst_value2': mock_schema({
|
||||
'type': "comma_delimited_list"}),
|
||||
'lst_value1': mock_schema({
|
||||
'type': "comma_delimited_list"}),
|
||||
'lst_value2': mock_schema({
|
||||
'type': "comma_delimited_list"}),
|
||||
'json_value1': mock_schema({
|
||||
'type': "json"}),
|
||||
'json_value2': mock_schema({
|
||||
'type': "json"})
|
||||
}
|
||||
|
||||
def test_merge_envs_with_param_default_merge_strategy(self):
|
||||
files = {'env_1': json.dumps(self.env_1),
|
||||
'env_2': json.dumps(self.env_2)}
|
||||
environment_files = ['env_1', 'env_2']
|
||||
|
||||
# Test
|
||||
env_util.merge_environments(environment_files, files, params)
|
||||
env_util.merge_environments(environment_files, files, self.params,
|
||||
self.param_schemata)
|
||||
|
||||
# Verify
|
||||
expected = {'parameters': {
|
||||
'p0': 'CORRECT',
|
||||
'p1': 'CORRECT',
|
||||
'p2': 'CORRECT',
|
||||
}}
|
||||
self.assertEqual(expected, params)
|
||||
'json_value1': {u'3': [u'str3', u'str4']},
|
||||
'json_value2': {u'4': [u'test3', u'test4']},
|
||||
'del_lst_value1': '5,6',
|
||||
'del_lst_value2': '7,8',
|
||||
'lst_value1': [5, 6],
|
||||
'lst_value2': [7, 8],
|
||||
'str_value1': u'string3',
|
||||
'str_value2': u'string4'},
|
||||
'resource_registry': {
|
||||
'test::R1': "OS::Heat::RandomString",
|
||||
'test::R2': "OS::Heat::None"}}
|
||||
self.assertEqual(expected, self.params)
|
||||
|
||||
def test_merge_environments_deep_merge(self):
|
||||
# Setup
|
||||
params = {'parameters': {
|
||||
'p0': 'CORRECT',
|
||||
'p1': ['CORRECT1'],
|
||||
'p2': {'A': ['CORRECT1', 'CORRECT2']},
|
||||
'p3': 'CORRECT1,CORRECT2'}
|
||||
}
|
||||
env_1 = '''
|
||||
{'parameters' : {
|
||||
'p1': ['CORRECT2', 'CORRECT3'],
|
||||
'p2': {'B': ['CORRECT3', 'CORRECT4'],
|
||||
'C': [CORRECT5]},
|
||||
'p3': 'CORRECT3,CORRECT4'
|
||||
}}'''
|
||||
env_2 = '''
|
||||
{'parameters': {
|
||||
'p2': {'C': ['CORRECT6']},
|
||||
'p3': 'CORRECT5,CORRECT6'
|
||||
}}'''
|
||||
|
||||
files = {'env_1': env_1, 'env_2': env_2}
|
||||
def test_merge_envs_with_specified_default(self):
|
||||
merge_strategies = {'default': 'deep_merge'}
|
||||
self.env_2['parameter_merge_strategies'] = merge_strategies
|
||||
files = {'env_1': json.dumps(self.env_1),
|
||||
'env_2': json.dumps(self.env_2)}
|
||||
environment_files = ['env_1', 'env_2']
|
||||
|
||||
# Test
|
||||
env_util.merge_environments(environment_files, files, params)
|
||||
env_util.merge_environments(environment_files, files, self.params,
|
||||
self.param_schemata)
|
||||
|
||||
# Verify
|
||||
# Does not work for comma_delimited_list.
|
||||
expected = {'parameters': {
|
||||
'p0': 'CORRECT',
|
||||
'p1': ['CORRECT1', 'CORRECT2', 'CORRECT3'],
|
||||
'p2': {'A': ['CORRECT1', 'CORRECT2'],
|
||||
'B': ['CORRECT3', 'CORRECT4'],
|
||||
'C': ['CORRECT5', 'CORRECT6']},
|
||||
'p3': 'CORRECT5,CORRECT6'
|
||||
}}
|
||||
self.assertEqual(expected, params)
|
||||
'json_value1': {u'3': [u'str3', u'str4'],
|
||||
u'1': [u'str1', u'str2']}, # added
|
||||
'json_value2': {u'4': [u'test3', u'test4'],
|
||||
u'2': [u'test1', u'test2']},
|
||||
'del_lst_value1': '1,2,5,6',
|
||||
'del_lst_value2': '3,4,7,8',
|
||||
'lst_value1': [1, 2, 5, 6], # added
|
||||
'lst_value2': [3, 4, 7, 8],
|
||||
'str_value1': u'string1string3',
|
||||
'str_value2': u'string2string4'},
|
||||
'resource_registry': {
|
||||
'test::R1': "OS::Heat::RandomString",
|
||||
'test::R2': "OS::Heat::None"}}
|
||||
self.assertEqual(expected, self.params)
|
||||
|
||||
def test_merge_envs_with_param_specific_merge_strategy(self):
|
||||
merge_strategies = {
|
||||
'default': "overwrite",
|
||||
'lst_value1': "merge",
|
||||
'json_value1': "deep_merge"}
|
||||
|
||||
self.env_2['parameter_merge_strategies'] = merge_strategies
|
||||
|
||||
files = {'env_1': json.dumps(self.env_1),
|
||||
'env_2': json.dumps(self.env_2)}
|
||||
environment_files = ['env_1', 'env_2']
|
||||
|
||||
# Test
|
||||
env_util.merge_environments(environment_files, files, self.params,
|
||||
self.param_schemata)
|
||||
|
||||
# Verify
|
||||
expected = {'parameters': {
|
||||
'json_value1': {u'3': [u'str3', u'str4'],
|
||||
u'1': [u'str1', u'str2']}, # added
|
||||
'json_value2': {u'4': [u'test3', u'test4']},
|
||||
'del_lst_value1': '5,6',
|
||||
'del_lst_value2': '7,8',
|
||||
'lst_value1': [1, 2, 5, 6], # added
|
||||
'lst_value2': [7, 8],
|
||||
'str_value1': u'string3',
|
||||
'str_value2': u'string4'},
|
||||
'resource_registry': {
|
||||
'test::R1': 'OS::Heat::RandomString',
|
||||
'test::R2': 'OS::Heat::None'}}
|
||||
self.assertEqual(expected, self.params)
|
||||
|
||||
def test_merge_environments_no_env_files(self):
|
||||
params = {'parameters': {'p0': 'CORRECT'}}
|
||||
env_1 = '''
|
||||
{'parameters' : {
|
||||
'p0': 'INCORRECT',
|
||||
}}'''
|
||||
|
||||
files = {'env_1': env_1}
|
||||
files = {'env_1': json.dumps(self.env_1)}
|
||||
|
||||
# Test - Should ignore env_1 in files
|
||||
env_util.merge_environments(None, files, params)
|
||||
env_util.merge_environments(None, files, self.params,
|
||||
self.param_schemata)
|
||||
|
||||
# Verify
|
||||
expected = {'parameters': {'p0': 'CORRECT'}}
|
||||
self.assertEqual(expected, params)
|
||||
expected = {'parameters': {
|
||||
'str_value1': "test1",
|
||||
'str_value2': "test2",
|
||||
'del_lst_value1': '',
|
||||
'del_lst_value2': '',
|
||||
'lst_value1': [],
|
||||
'lst_value2': [],
|
||||
'json_value1': {},
|
||||
'json_value2': {}},
|
||||
'resource_registry': {}}
|
||||
|
||||
self.assertEqual(expected, self.params)
|
||||
|
|
|
|||
Loading…
Reference in New Issue