diff --git a/hot/software-config/elements/heat-config-chef/README.rst b/hot/software-config/elements/heat-config-chef/README.rst new file mode 100644 index 0000000..bc77177 --- /dev/null +++ b/hot/software-config/elements/heat-config-chef/README.rst @@ -0,0 +1,36 @@ +A hook which invokes ``chef-client`` in local mode (chef zero) on the +provided configuration. + +Inputs: +------- +Inputs are attribute overrides. In order to format them correctly for +consumption, you need to explicitly declare each top-level section as an +input of type ``Json`` in your config resource. + +Additionally, there is a special input named ``environment`` of type +``String`` that you can use to specify which environment to use when +applying the config. You do not have to explicitly declare this input in +the config resource. + +Outputs: +-------- +If you need to capture specific outputs from your chef run, you should +specify the output name(s) as normal in your config. Then, your recipes +should write files to the directory specified by the ``heat_outputs_path`` +environment variable. The file name should match the name of the output +you are trying to capture. + +Options: +------------- + +kitchen : optional + A URL for a Git repository containing the desired recipes, roles, + environments and other configuration. + + This will be cloned into ``kitchen_path`` for use by chef. + +kitchen_path : default ``/var/lib/heat-config/heat-config-chef/kitchen`` + Instance-local path for the recipes, roles, environments, etc. + + If ``kitchen`` is not specified, this directory must be populated via + user-data, another software config, or other "manual" method. diff --git a/hot/software-config/elements/heat-config-chef/element-deps b/hot/software-config/elements/heat-config-chef/element-deps new file mode 100644 index 0000000..31d7aa5 --- /dev/null +++ b/hot/software-config/elements/heat-config-chef/element-deps @@ -0,0 +1 @@ +heat-config diff --git a/hot/software-config/elements/heat-config-chef/install.d/50-heat-config-hook-chef b/hot/software-config/elements/heat-config-chef/install.d/50-heat-config-hook-chef new file mode 100755 index 0000000..d34d56a --- /dev/null +++ b/hot/software-config/elements/heat-config-chef/install.d/50-heat-config-hook-chef @@ -0,0 +1,7 @@ +#!/bin/bash +set -x + +SCRIPTDIR=$(dirname $0) + +install-packages chef git +install -D -g root -o root -m 0755 ${SCRIPTDIR}/hook-chef.py /var/lib/heat-config/hooks/chef diff --git a/hot/software-config/elements/heat-config-chef/install.d/hook-chef.py b/hot/software-config/elements/heat-config-chef/install.d/hook-chef.py new file mode 100755 index 0000000..3a4d810 --- /dev/null +++ b/hot/software-config/elements/heat-config-chef/install.d/hook-chef.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json +import logging +import os +import shutil +import six +import subprocess +import sys + +DEPLOY_KEYS = ("deploy_server_id", + "deploy_action", + "deploy_stack_id", + "deploy_resource_name", + "deploy_signal_transport", + "deploy_signal_id", + "deploy_signal_verb") +WORKING_DIR = os.environ.get('HEAT_CHEF_WORKING', + '/var/lib/heat-config/heat-config-chef') +OUTPUTS_DIR = os.environ.get('HEAT_CHEF_OUTPUTS', + '/var/run/heat-config/heat-config-chef') + + +def prepare_dir(path): + if not os.path.isdir(path): + os.makedirs(path, 0o700) + + +def run_subproc(fn, **kwargs): + env = os.environ.copy() + for k, v in kwargs.items(): + env[six.text_type(k)] = v + try: + subproc = subprocess.Popen(fn, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env) + stdout, stderr = subproc.communicate() + except OSError as exc: + ret = -1 + stderr = six.text_type(exc) + stdout = "" + else: + ret = subproc.returncode + if not ret: + ret = 0 + return ret, stdout, stderr + + +def main(argv=sys.argv): + log = logging.getLogger('heat-config') + handler = logging.StreamHandler(sys.stderr) + handler.setFormatter( + logging.Formatter( + '[%(asctime)s] (%(name)s) [%(levelname)s] %(message)s')) + log.addHandler(handler) + log.setLevel('DEBUG') + + prepare_dir(OUTPUTS_DIR) + prepare_dir(WORKING_DIR) + os.chdir(WORKING_DIR) + + c = json.load(sys.stdin) + + client_config = ("log_level :debug\n" + "log_location STDOUT\n" + "local_mode true\n" + "chef_zero.enabled true") + + # configure/set up the kitchen + kitchen = c['options'].get('kitchen') + kitchen_path = c['options'].get('kitchen_path', os.path.join(WORKING_DIR, + "kitchen")) + cookbook_path = os.path.join(kitchen_path, "cookbooks") + role_path = os.path.join(kitchen_path, "roles") + environment_path = os.path.join(kitchen_path, "environments") + client_config += "\ncookbook_path '%s'" % cookbook_path + client_config += "\nrole_path '%s'" % role_path + client_config += "\nenvironment_path '%s'" % environment_path + if kitchen: + log.debug("Cloning kitchen from %s", kitchen) + # remove the existing kitchen on update so we get a fresh clone + dep_action = next((input['value'] for input in c['inputs'] + if input['name'] == "deploy_action"), None) + if dep_action == "UPDATE": + shutil.rmtree(kitchen_path, ignore_errors=True) + cmd = ["git", "clone", kitchen, kitchen_path] + ret, out, err = run_subproc(cmd) + if ret != 0: + log.error("Error cloning kitchen from %s into %s: %s", kitchen, + kitchen_path, err) + json.dump({'deploy_status_code': ret, + 'deploy_stdout': out, + 'deploy_stderr': err}, + sys.stdout) + return 0 + + # write the json attributes + ret, out, err = run_subproc(['hostname', '-f']) + if ret == 0: + fqdn = out.strip() + else: + err = "Could not determine hostname with hostname -f" + json.dump({'deploy_status_code': ret, + 'deploy_stdout': "", + 'deploy_stderr': err}, sys.stdout) + return 0 + node_config = {} + for input in c['inputs']: + if input['name'] == 'environment': + client_config += "\nenvironment '%s'" % input['value'] + elif input['name'] not in DEPLOY_KEYS: + node_config.update({input['name']: input['value']}) + node_config.update({"run_list": json.loads(c['config'])}) + node_path = os.path.join(WORKING_DIR, "node") + prepare_dir(node_path) + node_file = os.path.join(node_path, "%s.json" % fqdn) + with os.fdopen(os.open(node_file, os.O_CREAT | os.O_WRONLY, 0o600), + 'w') as f: + f.write(json.dumps(node_config, indent=4)) + client_config += "\nnode_path '%s'" % node_path + + # write out the completed client config + config_path = os.path.join(WORKING_DIR, "client.rb") + with os.fdopen(os.open(config_path, os.O_CREAT | os.O_WRONLY, 0o600), + 'w') as f: + f.write(client_config) + + # run chef + heat_outputs_path = os.path.join(OUTPUTS_DIR, c['id']) + cmd = ['chef-client', '-z', '--config', config_path, "-j", node_file] + ret, out, err = run_subproc(cmd, heat_outputs_path=heat_outputs_path) + resp = {'deploy_status_code': ret, + 'deploy_stdout': out, + 'deploy_stderr': err} + log.debug("Chef output: %s", out) + if err: + log.error("Chef return code %s:\n%s", ret, err) + for output in c.get('outputs', []): + output_name = output['name'] + try: + with open('%s.%s' % (heat_outputs_path, output_name)) as out: + resp[output_name] = out.read() + except IOError: + pass + json.dump(resp, sys.stdout) + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/tests/software_config/test_hook_chef.py b/tests/software_config/test_hook_chef.py new file mode 100644 index 0000000..8322037 --- /dev/null +++ b/tests/software_config/test_hook_chef.py @@ -0,0 +1,207 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy +import imp +import json +import logging +import mock +import StringIO +import sys + +from tests.software_config import common + +log = logging.getLogger('test_hook_chef') + + +@mock.patch("os.chdir") +@mock.patch("os.makedirs") +@mock.patch('subprocess.Popen') +class HookChefTest(common.RunScriptTest): + + data = { + 'id': 'fake_stack', + 'name': 'fake_resource_name', + 'group': 'chef', + 'inputs': [ + {'name': 'fooval', 'value': {'bar': 'baz'}}, + {'name': 'barval', 'value': {'foo': 'biff'}}, + {'name': "deploy_server_id", 'value': 'foo'}, + {'name': "deploy_action", 'value': 'foo'}, + {'name': "deploy_stack_id", 'value': 'foo'}, + {'name': "deploy_resource_name", 'value': 'foo'}, + {'name': "deploy_signal_transport", 'value': 'foo'}, + {'name': "deploy_signal_id", 'value': 'foo'}, + {'name': "deploy_signal_verb", 'value': 'foo'} + ], + 'options': {}, + 'outputs': [ + {'name': 'first_output'}, + {'name': 'second_output'} + ], + 'config': None + } + + def setUp(self): + super(HookChefTest, self).setUp() + self.hook_path = self.relative_path( + __file__, + '../..', + 'hot/software-config/elements', + 'heat-config-chef/install.d/hook-chef.py') + sys.stdin = StringIO.StringIO() + sys.stdout = StringIO.StringIO() + + def tearDown(self): + super(HookChefTest, self).tearDown() + sys.stdin = sys.__stdin__ + sys.stdout = sys.__stdout__ + + def get_module(self): + try: + imp.acquire_lock() + return imp.load_source("hook_chef", self.hook_path) + finally: + imp.release_lock() + + def test_hook(self, mock_popen, mock_mkdirs, mock_chdir): + data = copy.deepcopy(self.data) + data['config'] = '["recipe[apache]"]' + hook_chef = self.get_module() + sys.stdin.write(json.dumps(data)) + sys.stdin.seek(0) + mock_subproc = mock.Mock() + mock_popen.return_value = mock_subproc + mock_subproc.communicate.return_value = ("out", "err") + mock_subproc.returncode = 0 + with mock.patch("os.fdopen", mock.mock_open()) as mfdopen: + with mock.patch("os.open", mock.mock_open()): + hook_chef.main(json.dumps(data)) + exp_node = { + 'barval': {'foo': 'biff'}, + 'fooval': {u'bar': u'baz'}, + 'run_list': [u'recipe[apache]'] + } + exp_node = json.dumps(exp_node, indent=4) + exp_cfg = ("log_level :debug\n" + "log_location STDOUT\n" + "local_mode true\n" + "chef_zero.enabled true\n" + "cookbook_path '/var/lib/heat-config/" + "heat-config-chef/kitchen/cookbooks'\n" + "role_path '/var/lib/heat-config/" + "heat-config-chef/kitchen/roles'\n" + "environment_path '/var/lib/heat-config/" + "heat-config-chef/kitchen/environments'\n" + "node_path '/var/lib/heat-config/" + "heat-config-chef/node'") + mfdopen.return_value.write.assert_any_call(exp_cfg) + mfdopen.return_value.write.assert_any_call(exp_node) + calls = [ + mock.call(['hostname', '-f'], env=mock.ANY, stderr=mock.ANY, + stdout=mock.ANY), + mock.call([ + 'chef-client', '-z', '--config', + '/var/lib/heat-config/heat-config-chef/client.rb', '-j', + '/var/lib/heat-config/heat-config-chef/node/out.json'], + env=mock.ANY, stderr=mock.ANY, stdout=mock.ANY) + ] + mock_popen.assert_has_calls(calls, any_order=True) + self.assertEqual({"deploy_status_code": 0, + "deploy_stdout": "out", + "deploy_stderr": "err"}, + json.loads(sys.stdout.getvalue())) + + def test_hook_with_kitchen(self, mock_popen, mock_mkdirs, mock_chdir): + data = copy.deepcopy(self.data) + data['config'] = '["recipe[apache]"]' + data['options'] = { + "kitchen": "https://github.com/fake.git", + "kitchen_path": "/opt/heat/chef/kitchen" + } + sys.stdin.write(json.dumps(data)) + hook_chef = self.get_module() + sys.stdin.seek(0) + mock_subproc = mock.Mock() + mock_popen.return_value = mock_subproc + mock_subproc.communicate.return_value = ("out", "err") + mock_subproc.returncode = 0 + with mock.patch("os.fdopen", mock.mock_open()) as mfdopen: + with mock.patch("os.open", mock.mock_open()): + hook_chef.main(json.dumps(data)) + exp_cfg = ("log_level :debug\n" + "log_location STDOUT\n" + "local_mode true\n" + "chef_zero.enabled true\n" + "cookbook_path '/opt/heat/chef/kitchen/" + "cookbooks'\n" + "role_path '/opt/heat/chef/kitchen/roles'\n" + "environment_path '/opt/heat/chef/kitchen/" + "environments'\n" + "node_path '/var/lib/heat-config/heat-config-chef" + "/node'") + mfdopen.return_value.write.assert_any_call(exp_cfg) + calls = [ + mock.call(['git', 'clone', "https://github.com/fake.git", + "/opt/heat/chef/kitchen"], env=mock.ANY, + stderr=mock.ANY, stdout=mock.ANY), + mock.call(['hostname', '-f'], env=mock.ANY, stderr=mock.ANY, + stdout=mock.ANY), + mock.call([ + 'chef-client', '-z', '--config', + '/var/lib/heat-config/heat-config-chef/client.rb', '-j', + '/var/lib/heat-config/heat-config-chef/node/out.json'], + env=mock.ANY, stderr=mock.ANY, stdout=mock.ANY) + ] + mock_popen.assert_has_calls(calls, any_order=True) + self.assertEqual({"deploy_status_code": 0, + "deploy_stdout": "out", + "deploy_stderr": "err"}, + json.loads(sys.stdout.getvalue())) + + def test_hook_environment(self, mock_popen, mock_mkdirs, mock_chdir): + data = copy.deepcopy(self.data) + data['config'] = '["recipe[apache]"]' + data['inputs'].append({'name': 'environment', + 'value': 'production'}) + hook_chef = self.get_module() + sys.stdin.write(json.dumps(data)) + sys.stdin.seek(0) + mock_subproc = mock.Mock() + mock_popen.return_value = mock_subproc + mock_subproc.communicate.return_value = ("out", "err") + mock_subproc.returncode = 0 + with mock.patch("os.fdopen", mock.mock_open()) as mfdopen: + with mock.patch("os.open", mock.mock_open()): + hook_chef.main(json.dumps(data)) + exp_node = { + 'barval': {'foo': 'biff'}, + 'fooval': {u'bar': u'baz'}, + 'run_list': [u'recipe[apache]'] + } + exp_node = json.dumps(exp_node, indent=4) + exp_cfg = ("log_level :debug\n" + "log_location STDOUT\n" + "local_mode true\n" + "chef_zero.enabled true\n" + "cookbook_path '/var/lib/heat-config/" + "heat-config-chef/kitchen/cookbooks'\n" + "role_path '/var/lib/heat-config/" + "heat-config-chef/kitchen/roles'\n" + "environment_path '/var/lib/heat-config/" + "heat-config-chef/kitchen/environments'\n" + "environment 'production'\n" + "node_path '/var/lib/heat-config/" + "heat-config-chef/node'") + mfdopen.return_value.write.assert_any_call(exp_cfg) + mfdopen.return_value.write.assert_any_call(exp_node)