41e879ade7
Previously running cfn-json2yaml would generate valid YAML, but the
HOT file would not work. The key mappings it generates had 64 instead
of '64'. Since 64 in YAML is a number and not a string, the template
cannot find the key '64' as it is not a string value.
Turns out this issue was caused by simply forgetting to explicitly
re-add quotation marks after converting from a python object to YAML.
In the conversion code (heat/common/template_format.py), the json
string is first converted into a python object via yaml.load. Next
the python object gets converted to yaml via yaml.dump.
For example:
u'__00015__order__32': u'F17-i386-cfntools' in the python object gets
converted to __00015__order__32: F17-i386-cfntools in yaml.
Crucially in YAML, all strings are not explicity quoted except for
strings that contain only digits. For instance, numerical strings like
'32' will automatically get quoted when using yaml.dump
Normally this would be fine, but a subtle problem arises in the
next line of code:
yml = re.sub('__\d*__order__', '', yml)
By removing all instances of the order substring in the yaml, previous
alphanumeric strings become numeric-only strings. However, since this
step comes after the call to yaml.dump, quotes are not explicity set
anymore!
so for example:
__00015__order__32: F17-i386-cfntool becomes 32: F17-i386-cfntool
Then in YAML, the key 32 is now interpreted as number 32 and not
what we wanted, which is a string '32'
So to help fix this issue, replace all numeric-only keys with quoted
keys. For example:
32: F17-i386-cfntool becomes '32': F17-i386-cfntool
This helps to fix the issue of not finding the numerical keys in
the HOT yaml file.
Change-Id: I37208679f0699d088a7ca632a409d8675cad72c4
Closes-Bug: #1286380
Closes-Bug: #1467029
Closes-Bug: #1467026
127 lines
4.1 KiB
Python
127 lines
4.1 KiB
Python
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import itertools
|
|
import re
|
|
|
|
from oslo_config import cfg
|
|
from oslo_serialization import jsonutils
|
|
import six
|
|
import yaml
|
|
|
|
from heat.common import exception
|
|
from heat.common.i18n import _
|
|
|
|
cfg.CONF.import_opt('max_template_size', 'heat.common.config')
|
|
|
|
if hasattr(yaml, 'CSafeLoader'):
|
|
yaml_loader = yaml.CSafeLoader
|
|
else:
|
|
yaml_loader = yaml.SafeLoader
|
|
|
|
if hasattr(yaml, 'CSafeDumper'):
|
|
yaml_dumper = yaml.CSafeDumper
|
|
else:
|
|
yaml_dumper = yaml.SafeDumper
|
|
|
|
|
|
def _construct_yaml_str(self, node):
|
|
# Override the default string handling function
|
|
# to always return unicode objects
|
|
return self.construct_scalar(node)
|
|
yaml_loader.add_constructor(u'tag:yaml.org,2002:str', _construct_yaml_str)
|
|
# Unquoted dates like 2013-05-23 in yaml files get loaded as objects of type
|
|
# datetime.data which causes problems in API layer when being processed by
|
|
# openstack.common.jsonutils. Therefore, make unicode string out of timestamps
|
|
# until jsonutils can handle dates.
|
|
yaml_loader.add_constructor(u'tag:yaml.org,2002:timestamp',
|
|
_construct_yaml_str)
|
|
|
|
|
|
def simple_parse(tmpl_str):
|
|
try:
|
|
tpl = jsonutils.loads(tmpl_str)
|
|
except ValueError:
|
|
try:
|
|
tpl = yaml.load(tmpl_str, Loader=yaml_loader)
|
|
except yaml.YAMLError as yea:
|
|
yea = six.text_type(yea)
|
|
msg = _('Error parsing template: %s') % yea
|
|
raise ValueError(msg)
|
|
else:
|
|
if tpl is None:
|
|
tpl = {}
|
|
|
|
if not isinstance(tpl, dict):
|
|
raise ValueError(_('The template is not a JSON object '
|
|
'or YAML mapping.'))
|
|
|
|
return tpl
|
|
|
|
|
|
def parse(tmpl_str):
|
|
"""Takes a string and returns a dict containing the parsed structure.
|
|
|
|
This includes determination of whether the string is using the
|
|
JSON or YAML format.
|
|
"""
|
|
if len(tmpl_str) > cfg.CONF.max_template_size:
|
|
msg = (_('Template exceeds maximum allowed size (%s bytes)') %
|
|
cfg.CONF.max_template_size)
|
|
raise exception.RequestLimitExceeded(message=msg)
|
|
tpl = simple_parse(tmpl_str)
|
|
# Looking for supported version keys in the loaded template
|
|
if not ('HeatTemplateFormatVersion' in tpl
|
|
or 'heat_template_version' in tpl
|
|
or 'AWSTemplateFormatVersion' in tpl):
|
|
raise ValueError(_("Template format version not found."))
|
|
return tpl
|
|
|
|
|
|
def convert_json_to_yaml(json_str):
|
|
"""Convert AWS JSON template format to Heat YAML format.
|
|
|
|
:param json_str: a string containing the AWS JSON template format.
|
|
:returns: the equivalent string containing the Heat YAML format.
|
|
"""
|
|
|
|
# Replace AWS format version with Heat format version
|
|
json_str = re.sub('"AWSTemplateFormatVersion"\s*:\s*"[^"]+"\s*,',
|
|
'', json_str)
|
|
|
|
# insert a sortable order into the key to preserve file ordering
|
|
key_order = itertools.count()
|
|
|
|
def order_key(matchobj):
|
|
key = '%s"__%05d__order__%s" :' % (
|
|
matchobj.group(1),
|
|
next(key_order),
|
|
matchobj.group(2))
|
|
return key
|
|
key_re = re.compile('(\s*)"([^"]+)"\s*:')
|
|
json_str = key_re.sub(order_key, json_str)
|
|
|
|
# parse the string as json to a python structure
|
|
tpl = yaml.load(json_str, Loader=yaml_loader)
|
|
|
|
# dump python structure to yaml
|
|
tpl["HeatTemplateFormatVersion"] = '2012-12-12'
|
|
yml = yaml.dump(tpl, Dumper=yaml_dumper)
|
|
|
|
# remove ordering from key names
|
|
yml = re.sub('__\d*__order__', '', yml)
|
|
|
|
# convert integer keys back to string
|
|
yml = re.sub('([\s,{])(\d+)(\s*):', r"\1'\2'\3:", yml)
|
|
return yml
|