openstack/heat-templates
Code Issues Proposed changes

Chef-Zero software config hook

Browse Source 
Implements a software config hook for using chef-zero, an in-memory,
standalone chef service. Chef-Zero has replaced chef-solo as the
preferred serverless implementation, therfore this patch replaces
the previous chef-solo patch at https://review.openstack.org/#/c/80229/.

Change-Id: Ic93b2e67be0c07e27d26ed2c6677c5a0f1b42e07
Closes-Bug: #1279062
This commit is contained in:
Randall Burt 2015-12-23 15:51:21 -06:00
parent 2a7fe20efa
commit 2534497008
6 changed files with 488 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
hot/software-config/elements/heat-config-chef/README.rst Normal file
View File

@ -0,0 +1,36 @@
A hook which invokes ``chef-client`` in local mode (chef zero) on the
provided configuration.
Inputs:
-------
Inputs are attribute overrides. In order to format them correctly for
consumption, you need to explicitly declare each top-level section as an
input of type ``Json`` in your config resource.
Additionally, there is a special input named ``environment`` of type
``String`` that you can use to specify which environment to use when
applying the config. You do not have to explicitly declare this input in
the config resource.
Outputs:
--------
If you need to capture specific outputs from your chef run, you should
specify the output name(s) as normal in your config. Then, your recipes
should write files to the directory specified by the ``heat_outputs_path``
environment variable. The file name should match the name of the output
you are trying to capture.
Options:
-------------
kitchen : optional
A URL for a Git repository containing the desired recipes, roles,
environments and other configuration.
This will be cloned into ``kitchen_path`` for use by chef.
kitchen_path : default ``/var/lib/heat-config/heat-config-chef/kitchen``
Instance-local path for the recipes, roles, environments, etc.
If ``kitchen`` is not specified, this directory must be populated via
user-data, another software config, or other "manual" method.

1
hot/software-config/elements/heat-config-chef/element-deps Normal file
View File

@ -0,0 +1 @@
heat-config

7
hot/software-config/elements/heat-config-chef/install.d/50-heat-config-hook-chef Executable file
View File

@ -0,0 +1,7 @@
#!/bin/bash
set -x
SCRIPTDIR=$(dirname $0)
install-packages chef git
install -D -g root -o root -m 0755 ${SCRIPTDIR}/hook-chef.py /var/lib/heat-config/hooks/chef

161
hot/software-config/elements/heat-config-chef/install.d/hook-chef.py Executable file
View File

@ -0,0 +1,161 @@
#!/usr/bin/env python
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import json
import logging
import os
import shutil
import six
import subprocess
import sys
DEPLOY_KEYS = ("deploy_server_id",
"deploy_action",
"deploy_stack_id",
"deploy_resource_name",
"deploy_signal_transport",
"deploy_signal_id",
"deploy_signal_verb")
WORKING_DIR = os.environ.get('HEAT_CHEF_WORKING',
'/var/lib/heat-config/heat-config-chef')
OUTPUTS_DIR = os.environ.get('HEAT_CHEF_OUTPUTS',
'/var/run/heat-config/heat-config-chef')
def prepare_dir(path):
if not os.path.isdir(path):
os.makedirs(path, 0o700)
def run_subproc(fn, **kwargs):
env = os.environ.copy()
for k, v in kwargs.items():
env[six.text_type(k)] = v
try:
subproc = subprocess.Popen(fn, stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
stdout, stderr = subproc.communicate()
except OSError as exc:
ret = -1
stderr = six.text_type(exc)
stdout = ""
else:
ret = subproc.returncode
if not ret:
ret = 0
return ret, stdout, stderr
def main(argv=sys.argv):
log = logging.getLogger('heat-config')
handler = logging.StreamHandler(sys.stderr)
handler.setFormatter(
logging.Formatter(
'[%(asctime)s] (%(name)s) [%(levelname)s] %(message)s'))
log.addHandler(handler)
log.setLevel('DEBUG')
prepare_dir(OUTPUTS_DIR)
prepare_dir(WORKING_DIR)
os.chdir(WORKING_DIR)
c = json.load(sys.stdin)
client_config = ("log_level :debug\n"
"log_location STDOUT\n"
"local_mode true\n"
"chef_zero.enabled true")
# configure/set up the kitchen
kitchen = c['options'].get('kitchen')
kitchen_path = c['options'].get('kitchen_path', os.path.join(WORKING_DIR,
"kitchen"))
cookbook_path = os.path.join(kitchen_path, "cookbooks")
role_path = os.path.join(kitchen_path, "roles")
environment_path = os.path.join(kitchen_path, "environments")
client_config += "\ncookbook_path '%s'" % cookbook_path
client_config += "\nrole_path '%s'" % role_path
client_config += "\nenvironment_path '%s'" % environment_path
if kitchen:
log.debug("Cloning kitchen from %s", kitchen)
# remove the existing kitchen on update so we get a fresh clone
dep_action = next((input['value'] for input in c['inputs']
if input['name'] == "deploy_action"), None)
if dep_action == "UPDATE":
shutil.rmtree(kitchen_path, ignore_errors=True)
cmd = ["git", "clone", kitchen, kitchen_path]
ret, out, err = run_subproc(cmd)
if ret != 0:
log.error("Error cloning kitchen from %s into %s: %s", kitchen,
kitchen_path, err)
json.dump({'deploy_status_code': ret,
'deploy_stdout': out,
'deploy_stderr': err},
sys.stdout)
return 0
# write the json attributes
ret, out, err = run_subproc(['hostname', '-f'])
if ret == 0:
fqdn = out.strip()
else:
err = "Could not determine hostname with hostname -f"
json.dump({'deploy_status_code': ret,
'deploy_stdout': "",
'deploy_stderr': err}, sys.stdout)
return 0
node_config = {}
for input in c['inputs']:
if input['name'] == 'environment':
client_config += "\nenvironment '%s'" % input['value']
elif input['name'] not in DEPLOY_KEYS:
node_config.update({input['name']: input['value']})
node_config.update({"run_list": json.loads(c['config'])})
node_path = os.path.join(WORKING_DIR, "node")
prepare_dir(node_path)
node_file = os.path.join(node_path, "%s.json" % fqdn)
with os.fdopen(os.open(node_file, os.O_CREAT | os.O_WRONLY, 0o600),
'w') as f:
f.write(json.dumps(node_config, indent=4))
client_config += "\nnode_path '%s'" % node_path
# write out the completed client config
config_path = os.path.join(WORKING_DIR, "client.rb")
with os.fdopen(os.open(config_path, os.O_CREAT | os.O_WRONLY, 0o600),
'w') as f:
f.write(client_config)
# run chef
heat_outputs_path = os.path.join(OUTPUTS_DIR, c['id'])
cmd = ['chef-client', '-z', '--config', config_path, "-j", node_file]
ret, out, err = run_subproc(cmd, heat_outputs_path=heat_outputs_path)
resp = {'deploy_status_code': ret,
'deploy_stdout': out,
'deploy_stderr': err}
log.debug("Chef output: %s", out)
if err:
log.error("Chef return code %s:\n%s", ret, err)
for output in c.get('outputs', []):
output_name = output['name']
try:
with open('%s.%s' % (heat_outputs_path, output_name)) as out:
resp[output_name] = out.read()
except IOError:
pass
json.dump(resp, sys.stdout)
if __name__ == '__main__':
sys.exit(main(sys.argv))

76
hot/software-config/example-templates/example-chef-template.yaml Normal file
View File

@ -0,0 +1,76 @@
heat_template_version: 2014-10-16
parameters:
key_name:
type: string
flavor:
type: string
default: m1.small
image:
type: string
default: fedora-software-config
resources:
the_sg:
type: OS::Neutron::SecurityGroup
properties:
name: the_sg
description: Ping and SSH
rules:
- protocol: icmp
- protocol: tcp
port_range_min: 22
port_range_max: 22
config:
type: OS::Heat::StructuredConfig
properties:
group: chef
inputs:
- name: nginx-pkg
type: Json
config:
- "recipe[nginx-pkg]"
options:
kitchen: "https://github.com/st-isidore-de-seville/cookbook-nginx-pkg.git"
kitchen_path: "/opt/heat/chef"
deployment:
type: OS::Heat::StructuredDeployment
properties:
config:
get_resource: structured_config
server:
get_resource: server
input_values:
nginx-pkg:
package:
name: nginx
server:
type: OS::Nova::Server
properties:
image: {get_param: image}
flavor: {get_param: flavor}
key_name: {get_param: key_name}
security_groups:
- {get_resource: the_sg}
user_data_format: SOFTWARE_CONFIG
outputs:
status_code_deployment:
value:
get_attr: [deployment, deploy_status_code]
stdout:
value:
get_attr: [deployment, deploy_stdout]
stderr:
value:
get_attr: [deployment, deploy_stderr]

207
tests/software_config/test_hook_chef.py Normal file
View File

@ -0,0 +1,207 @@
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import copy
import imp
import json
import logging
import mock
import StringIO
import sys
from tests.software_config import common
log = logging.getLogger('test_hook_chef')
@mock.patch("os.chdir")
@mock.patch("os.makedirs")
@mock.patch('subprocess.Popen')
class HookChefTest(common.RunScriptTest):
data = {
'id': 'fake_stack',
'name': 'fake_resource_name',
'group': 'chef',
'inputs': [
{'name': 'fooval', 'value': {'bar': 'baz'}},
{'name': 'barval', 'value': {'foo': 'biff'}},
{'name': "deploy_server_id", 'value': 'foo'},
{'name': "deploy_action", 'value': 'foo'},
{'name': "deploy_stack_id", 'value': 'foo'},
{'name': "deploy_resource_name", 'value': 'foo'},
{'name': "deploy_signal_transport", 'value': 'foo'},
{'name': "deploy_signal_id", 'value': 'foo'},
{'name': "deploy_signal_verb", 'value': 'foo'}
],
'options': {},
'outputs': [
{'name': 'first_output'},
{'name': 'second_output'}
],
'config': None
}
def setUp(self):
super(HookChefTest, self).setUp()
self.hook_path = self.relative_path(
__file__,
'../..',
'hot/software-config/elements',
'heat-config-chef/install.d/hook-chef.py')
sys.stdin = StringIO.StringIO()
sys.stdout = StringIO.StringIO()
def tearDown(self):
super(HookChefTest, self).tearDown()
sys.stdin = sys.__stdin__
sys.stdout = sys.__stdout__
def get_module(self):
try:
imp.acquire_lock()
return imp.load_source("hook_chef", self.hook_path)
finally:
imp.release_lock()
def test_hook(self, mock_popen, mock_mkdirs, mock_chdir):
data = copy.deepcopy(self.data)
data['config'] = '["recipe[apache]"]'
hook_chef = self.get_module()
sys.stdin.write(json.dumps(data))
sys.stdin.seek(0)
mock_subproc = mock.Mock()
mock_popen.return_value = mock_subproc
mock_subproc.communicate.return_value = ("out", "err")
mock_subproc.returncode = 0
with mock.patch("os.fdopen", mock.mock_open()) as mfdopen:
with mock.patch("os.open", mock.mock_open()):
hook_chef.main(json.dumps(data))
exp_node = {
'barval': {'foo': 'biff'},
'fooval': {u'bar': u'baz'},
'run_list': [u'recipe[apache]']
}
exp_node = json.dumps(exp_node, indent=4)
exp_cfg = ("log_level :debug\n"
"log_location STDOUT\n"
"local_mode true\n"
"chef_zero.enabled true\n"
"cookbook_path '/var/lib/heat-config/"
"heat-config-chef/kitchen/cookbooks'\n"
"role_path '/var/lib/heat-config/"
"heat-config-chef/kitchen/roles'\n"
"environment_path '/var/lib/heat-config/"
"heat-config-chef/kitchen/environments'\n"
"node_path '/var/lib/heat-config/"
"heat-config-chef/node'")
mfdopen.return_value.write.assert_any_call(exp_cfg)
mfdopen.return_value.write.assert_any_call(exp_node)
calls = [
mock.call(['hostname', '-f'], env=mock.ANY, stderr=mock.ANY,
stdout=mock.ANY),
mock.call([
'chef-client', '-z', '--config',
'/var/lib/heat-config/heat-config-chef/client.rb', '-j',
'/var/lib/heat-config/heat-config-chef/node/out.json'],
env=mock.ANY, stderr=mock.ANY, stdout=mock.ANY)
]
mock_popen.assert_has_calls(calls, any_order=True)
self.assertEqual({"deploy_status_code": 0,
"deploy_stdout": "out",
"deploy_stderr": "err"},
json.loads(sys.stdout.getvalue()))
def test_hook_with_kitchen(self, mock_popen, mock_mkdirs, mock_chdir):
data = copy.deepcopy(self.data)
data['config'] = '["recipe[apache]"]'
data['options'] = {
"kitchen": "https://github.com/fake.git",
"kitchen_path": "/opt/heat/chef/kitchen"
}
sys.stdin.write(json.dumps(data))
hook_chef = self.get_module()
sys.stdin.seek(0)
mock_subproc = mock.Mock()
mock_popen.return_value = mock_subproc
mock_subproc.communicate.return_value = ("out", "err")
mock_subproc.returncode = 0
with mock.patch("os.fdopen", mock.mock_open()) as mfdopen:
with mock.patch("os.open", mock.mock_open()):
hook_chef.main(json.dumps(data))
exp_cfg = ("log_level :debug\n"
"log_location STDOUT\n"
"local_mode true\n"
"chef_zero.enabled true\n"
"cookbook_path '/opt/heat/chef/kitchen/"
"cookbooks'\n"
"role_path '/opt/heat/chef/kitchen/roles'\n"
"environment_path '/opt/heat/chef/kitchen/"
"environments'\n"
"node_path '/var/lib/heat-config/heat-config-chef"
"/node'")
mfdopen.return_value.write.assert_any_call(exp_cfg)
calls = [
mock.call(['git', 'clone', "https://github.com/fake.git",
"/opt/heat/chef/kitchen"], env=mock.ANY,
stderr=mock.ANY, stdout=mock.ANY),
mock.call(['hostname', '-f'], env=mock.ANY, stderr=mock.ANY,
stdout=mock.ANY),
mock.call([
'chef-client', '-z', '--config',
'/var/lib/heat-config/heat-config-chef/client.rb', '-j',
'/var/lib/heat-config/heat-config-chef/node/out.json'],
env=mock.ANY, stderr=mock.ANY, stdout=mock.ANY)
]
mock_popen.assert_has_calls(calls, any_order=True)
self.assertEqual({"deploy_status_code": 0,
"deploy_stdout": "out",
"deploy_stderr": "err"},
json.loads(sys.stdout.getvalue()))
def test_hook_environment(self, mock_popen, mock_mkdirs, mock_chdir):
data = copy.deepcopy(self.data)
data['config'] = '["recipe[apache]"]'
data['inputs'].append({'name': 'environment',
'value': 'production'})
hook_chef = self.get_module()
sys.stdin.write(json.dumps(data))
sys.stdin.seek(0)
mock_subproc = mock.Mock()
mock_popen.return_value = mock_subproc
mock_subproc.communicate.return_value = ("out", "err")
mock_subproc.returncode = 0
with mock.patch("os.fdopen", mock.mock_open()) as mfdopen:
with mock.patch("os.open", mock.mock_open()):
hook_chef.main(json.dumps(data))
exp_node = {
'barval': {'foo': 'biff'},
'fooval': {u'bar': u'baz'},
'run_list': [u'recipe[apache]']
}
exp_node = json.dumps(exp_node, indent=4)
exp_cfg = ("log_level :debug\n"
"log_location STDOUT\n"
"local_mode true\n"
"chef_zero.enabled true\n"
"cookbook_path '/var/lib/heat-config/"
"heat-config-chef/kitchen/cookbooks'\n"
"role_path '/var/lib/heat-config/"
"heat-config-chef/kitchen/roles'\n"
"environment_path '/var/lib/heat-config/"
"heat-config-chef/kitchen/environments'\n"
"environment 'production'\n"
"node_path '/var/lib/heat-config/"
"heat-config-chef/node'")
mfdopen.return_value.write.assert_any_call(exp_cfg)
mfdopen.return_value.write.assert_any_call(exp_node)