A hook which writes hiera datafiles.

This patch creates a hook that writes hiera.yaml and datafiles
to disk. This approach works well for users (like TripleO)
who prefer to maintain Hiera in a more composable manner with
Heat structured config resources rather than specifying all
the Hiera values for each puppet config resource.

Partial-bug: #1596373

Change-Id: I21639f6aadabf9e49f40d1bb0b1d0edcfc4dbc5e

hot/software-config/elements/heat-config-hiera/README.rst

@@ -0,0 +1,25 @@
+A hook which helps write hiera files to disk and creates
+the hiera.yaml to order them. This is typically used alongside
+of the puppet hook to generate Hiera in a more composable manner.
+
+Example:
+
+  ComputeConfig:
+    type: OS::Heat::StructuredConfig
+    properties:
+      group: hiera
+      config:
+        hierarchy:
+          - compute
+        datafiles:
+          compute:
+            debug: true
+            db_connection: foo:/bar
+            # customized hiera goes here...
+
+This would write out:
+
+ 1) An /etc/hiera.yaml config file with compute in the hierarchy.
+
+ 2) An /etc/puppet/hieradata/compute.json file loaded with the
+    custom hiera data.

hot/software-config/elements/heat-config-hiera/element-deps

@@ -0,0 +1 @@
+heat-config

hot/software-config/elements/heat-config-hiera/install.d/50-heat-config-hook-hiera

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -x
+
+SCRIPTDIR=$(dirname $0)
+
+install-packages hiera
+install -D -g root -o root -m 0755 ${SCRIPTDIR}/hook-hiera.py /var/lib/heat-config/hooks/hiera
+
+ln -f -s /etc/puppet/hiera.yaml /etc/hiera.yaml

hot/software-config/elements/heat-config-hiera/install.d/hook-hiera.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import json
+import logging
+import os
+import sys
+
+
+HIERA_DATADIR = os.environ.get('HEAT_PUPPET_HIERA_DATADIR',
+                               '/etc/puppet/hieradata')
+HIERA_CONFIG = os.environ.get('HEAT_HIERA_CONFIG', '/etc/puppet/hiera.yaml')
+
+HIERA_CONFIG_BASE = """
+---
+:backends:
+  - json
+:json:
+  :datadir: %(datadir)s
+:hierarchy:
+""" % {'datadir': HIERA_DATADIR}
+
+
+def prepare_dir(path):
+    if not os.path.isdir(path):
+        os.makedirs(path, 0o700)
+
+
+def main(argv=sys.argv):
+    log = logging.getLogger('heat-config')
+    handler = logging.StreamHandler(sys.stderr)
+    handler.setFormatter(
+        logging.Formatter(
+            '[%(asctime)s] (%(name)s) [%(levelname)s] %(message)s'))
+    log.addHandler(handler)
+    log.setLevel('DEBUG')
+
+    c = json.load(sys.stdin)['config']
+
+    prepare_dir(HIERA_DATADIR)
+
+    hiera_config_file = os.path.join(HIERA_CONFIG)
+
+    # allow the end user to order the hiera config as they wish
+    if 'hierarchy' in c:
+        with os.fdopen(os.open(hiera_config_file,
+                               os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600),
+                       'w') as config_file:
+            config_file.write(HIERA_CONFIG_BASE)
+            for item in c['hierarchy']:
+                config_file.write('  - %s\n' % item)
+
+    # write out the datafiles as YAML
+    if 'datafiles' in c:
+        for name, data in c['datafiles'].iteritems():
+            hiera_data = os.path.join(HIERA_DATADIR, '%s.json' % name)
+            with os.fdopen(os.open(hiera_data,
+                                   os.O_CREAT | os.O_TRUNC | os.O_WRONLY,
+                                   0o600),
+                           'w') as hiera_data_file:
+                json.dump(data, hiera_data_file, indent=4, sort_keys=True)
+
+    response = {
+        'deploy_stdout': '',
+        'deploy_stderr': '',
+        'deploy_status_code': 0,
+    }
+
+    json.dump(response, sys.stdout)
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv))

tests/software_config/test_heat_config.py

@@ -26,7 +26,7 @@ from tests.software_config import common
 class HeatConfigTest(common.RunScriptTest):
 
     fake_hooks = ['cfn-init', 'chef', 'puppet', 'salt', 'script',
-                  'apply-config']
+                  'apply-config', 'hiera']
 
     data = [
         {
@@ -70,6 +70,11 @@ class HeatConfigTest(common.RunScriptTest):
             'group': 'apply-config',
             'inputs': [{'name': 'foo', 'value': 'bar'}],
             'config': 'six'
+        }, {
+            'id': '7777',
+            'group': 'hiera',
+            'inputs': [],
+            'config': 'seven'
         }, {
             'id': '9999',
             'group': 'no-such-hook',
@@ -104,6 +109,11 @@ class HeatConfigTest(common.RunScriptTest):
             'deploy_stderr': 'A bad thing happened',
             'deploy_stdout': 'stdout'
         },
+        'hiera': {
+            'deploy_status_code': '0',
+            'deploy_stderr': 'stderr',
+            'deploy_stdout': 'stdout'
+        },
         'apply-config': {
             'deploy_status_code': '0',
             'deploy_stderr': 'stderr',

tests/software_config/test_hook_hiera.py

@@ -0,0 +1,85 @@
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import fixtures
+import json
+import logging
+import os
+import tempfile
+import yaml
+
+from tests.software_config import common
+
+log = logging.getLogger('test_hook_hiera_config')
+
+HIERA_CONFIG_BASE = """
+---
+:backends:
+  - json
+:json:
+  :datadir: %(datadir)s
+:hierarchy:
+  - %(datafile)s
+"""
+
+
+class HookHieraTest(common.RunScriptTest):
+
+    data = {
+        'id': 'test_hiera',
+        'name': 'fake_resource_name',
+        'group': 'hiera',
+        'config': {
+            'hierarchy': ['compute'],
+            'datafiles': {
+                'compute': {'foo': 'bar'}
+            }
+        }
+    }
+
+    def setUp(self):
+        super(HookHieraTest, self).setUp()
+        self.hook_path = self.relative_path(
+            __file__,
+            '../..',
+            'hot/software-config/elements',
+            'heat-config-hiera/install.d/hook-hiera.py')
+
+        self.hieradata_dir = self.useFixture(fixtures.TempDir()).join()
+        self.conf = tempfile.NamedTemporaryFile(mode='w', delete=False).name
+        os.unlink(self.conf)
+
+        self.env = os.environ.copy()
+        self.env.update({
+            'HEAT_HIERA_CONFIG': self.conf,
+            'HEAT_PUPPET_HIERA_DATADIR': self.hieradata_dir,
+        })
+
+    def test_hook(self):
+
+        returncode, stdout, stderr = self.run_cmd(
+            [self.hook_path], self.env, json.dumps(self.data))
+
+        self.assertEqual(0, returncode, stderr)
+        ret = yaml.safe_load(stdout)
+        self.assertIsNotNone(ret['deploy_stderr'])
+        self.assertEqual('', ret['deploy_stdout'])
+        self.assertEqual(0, ret['deploy_status_code'])
+
+        conf_data = HIERA_CONFIG_BASE % {'datadir': self.hieradata_dir,
+                                         'datafile': 'compute'}
+        with open(self.conf) as conf_file:
+            self.assertEqual(conf_data, conf_file.read())
+
+        with open(os.path.join(self.hieradata_dir, 'compute.json')) as data:
+            self.assertEqual("{\n    \"foo\": \"bar\"\n}", data.read())