Module for converting JSON to YAML, and parsing both

convert_json_to_yaml is a utility function used for tests and file conversion
utilities.

parse_to_template will take any string, infer the format, and parse to a
python structure. Currently it assumes the file is JSON if it starts with '{'
otherwise it attempts to parse it as YAML.

Change-Id: If15ccdaf912693f76b74bb1fe879145af1cb36b1

heat/engine/format.py

@@ -0,0 +1,74 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import re
+import yaml
+import json
+
+
+def _construct_yaml_str(self, node):
+    # Override the default string handling function
+    # to always return unicode objects
+    return self.construct_scalar(node)
+yaml.Loader.add_constructor(u'tag:yaml.org,2002:str', _construct_yaml_str)
+yaml.SafeLoader.add_constructor(u'tag:yaml.org,2002:str', _construct_yaml_str)
+
+
+def parse_to_template(tmpl_str):
+    '''
+    Takes a string and returns a dict containing the parsed structure.
+    This includes determination of whether the string is using the
+    JSON or YAML format.
+    '''
+    if tmpl_str.startswith('{'):
+        return json.loads(tmpl_str)
+    try:
+        return yaml.load(tmpl_str)
+    except yaml.scanner.ScannerError as e:
+        raise ValueError(e)
+
+
+def convert_json_to_yaml(json_str):
+    '''Convert a string containing the AWS JSON template format
+    to an equivalent string containing the Heat YAML format.'''
+
+    global key_order
+    # Replace AWS format version with Heat format version
+    json_str = re.sub('"AWSTemplateFormatVersion"\s*:\s*"[^"]+"\s*,',
+        '', json_str)
+
+    # insert a sortable order into the key to preserve file ordering
+    key_order = 0
+
+    def order_key(matchobj):
+        global key_order
+        key = '%s"__%05d__order__%s" :' % (
+            matchobj.group(1),
+            key_order,
+            matchobj.group(2))
+        key_order = key_order + 1
+        return key
+    key_re = re.compile('^(\s*)"([^"]+)"\s*:', re.M)
+    json_str = key_re.sub(order_key, json_str)
+
+    # parse the string as json to a python structure
+    tpl = yaml.load(json_str)
+
+    # dump python structure to yaml
+    yml = "HeatTemplateFormatVersion: '2012-12-12'\n" + yaml.safe_dump(tpl)
+
+    # remove ordering from key names
+    yml = re.sub('__\d*__order__', '', yml)
+    return yml

heat/tests/test_format.py

@@ -0,0 +1,73 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import json
+import mox
+import nose
+from nose.plugins.attrib import attr
+import os
+import re
+import unittest
+import yaml
+
+from heat.engine import format
+from heat.engine import parser
+
+
+@attr(tag=['unit'])
+class JsonToYamlTest(unittest.TestCase):
+
+    def setUp(self):
+        self.expected_test_count = 29
+        self.longMessage = True
+
+    def test_convert_all_templates(self):
+        path = os.path.dirname(os.path.realpath(__file__)).\
+            replace('heat/tests', 'templates')
+
+        template_test_count = 0
+        for (json_str,
+            yml_str,
+            file_name) in self.convert_all_json_to_yaml(path):
+
+            self.compare_json_vs_yaml(json_str, yml_str, file_name)
+            template_test_count += 1
+
+        self.assertTrue(template_test_count >= self.expected_test_count,
+            'Expected at least %d templates to be tested' %
+            self.expected_test_count)
+
+    def compare_json_vs_yaml(self, json_str, yml_str, file_name):
+        yml = format.parse_to_template(yml_str)
+
+        self.assertEqual(u'2012-12-12', yml[u'HeatTemplateFormatVersion'],
+            file_name)
+        self.assertFalse(u'AWSTemplateFormatVersion' in yml, file_name)
+        del(yml[u'HeatTemplateFormatVersion'])
+
+        jsn = format.parse_to_template(json_str)
+        if u'AWSTemplateFormatVersion' in jsn:
+            del(jsn[u'AWSTemplateFormatVersion'])
+
+        self.assertEqual(yml, jsn, file_name)
+
+    def convert_all_json_to_yaml(self, dirpath):
+        for path in os.listdir(dirpath):
+            if not path.endswith('.template') and not path.endswith('.json'):
+                continue
+            f = open(os.path.join(dirpath, path), 'r')
+            json_str = f.read()
+
+            yml_str = format.convert_json_to_yaml(json_str)
+            yield (json_str, yml_str, f.name)