Chef-Zero software config hook

Implements a software config hook for using chef-zero, an in-memory,
standalone chef service. Chef-Zero has replaced chef-solo as the
preferred serverless implementation, therfore this patch replaces
the previous chef-solo patch at https://review.openstack.org/#/c/80229/.

Change-Id: Ic93b2e67be0c07e27d26ed2c6677c5a0f1b42e07
Closes-Bug: #1279062

hot/software-config/elements/heat-config-chef/README.rst

@@ -0,0 +1,36 @@
+A hook which invokes ``chef-client`` in local mode (chef zero) on the
+provided configuration.
+
+Inputs:
+-------
+Inputs are attribute overrides. In order to format them correctly for
+consumption, you need to explicitly declare each top-level section as an
+input of type ``Json`` in your config resource.
+
+Additionally, there is a special input named ``environment`` of type
+``String`` that you can use to specify which environment to use when
+applying the config. You do not have to explicitly declare this input in
+the config resource.
+
+Outputs:
+--------
+If you need to capture specific outputs from your chef run, you should
+specify the output name(s) as normal in your config. Then, your recipes
+should write files to the directory specified by the ``heat_outputs_path``
+environment variable. The file name should match the name of the output
+you are trying to capture.
+
+Options:
+-------------
+
+kitchen : optional
+    A URL for a Git repository containing the desired recipes, roles,
+    environments and other configuration.
+
+    This will be cloned into ``kitchen_path`` for use by chef.
+
+kitchen_path : default ``/var/lib/heat-config/heat-config-chef/kitchen``
+    Instance-local path for the recipes, roles, environments, etc.
+
+    If ``kitchen`` is not specified, this directory must be populated via
+    user-data, another software config, or other "manual" method.

hot/software-config/elements/heat-config-chef/element-deps

@@ -0,0 +1 @@
+heat-config

hot/software-config/elements/heat-config-chef/install.d/50-heat-config-hook-chef

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -x
+
+SCRIPTDIR=$(dirname $0)
+
+install-packages chef git
+install -D -g root -o root -m 0755 ${SCRIPTDIR}/hook-chef.py /var/lib/heat-config/hooks/chef

hot/software-config/elements/heat-config-chef/install.d/hook-chef.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import json
+import logging
+import os
+import shutil
+import six
+import subprocess
+import sys
+
+DEPLOY_KEYS = ("deploy_server_id",
+               "deploy_action",
+               "deploy_stack_id",
+               "deploy_resource_name",
+               "deploy_signal_transport",
+               "deploy_signal_id",
+               "deploy_signal_verb")
+WORKING_DIR = os.environ.get('HEAT_CHEF_WORKING',
+                             '/var/lib/heat-config/heat-config-chef')
+OUTPUTS_DIR = os.environ.get('HEAT_CHEF_OUTPUTS',
+                             '/var/run/heat-config/heat-config-chef')
+
+
+def prepare_dir(path):
+    if not os.path.isdir(path):
+        os.makedirs(path, 0o700)
+
+
+def run_subproc(fn, **kwargs):
+    env = os.environ.copy()
+    for k, v in kwargs.items():
+        env[six.text_type(k)] = v
+    try:
+        subproc = subprocess.Popen(fn, stdout=subprocess.PIPE,
+                                   stderr=subprocess.PIPE,
+                                   env=env)
+        stdout, stderr = subproc.communicate()
+    except OSError as exc:
+        ret = -1
+        stderr = six.text_type(exc)
+        stdout = ""
+    else:
+        ret = subproc.returncode
+    if not ret:
+        ret = 0
+    return ret, stdout, stderr
+
+
+def main(argv=sys.argv):
+    log = logging.getLogger('heat-config')
+    handler = logging.StreamHandler(sys.stderr)
+    handler.setFormatter(
+        logging.Formatter(
+            '[%(asctime)s] (%(name)s) [%(levelname)s] %(message)s'))
+    log.addHandler(handler)
+    log.setLevel('DEBUG')
+
+    prepare_dir(OUTPUTS_DIR)
+    prepare_dir(WORKING_DIR)
+    os.chdir(WORKING_DIR)
+
+    c = json.load(sys.stdin)
+
+    client_config = ("log_level :debug\n"
+                     "log_location STDOUT\n"
+                     "local_mode true\n"
+                     "chef_zero.enabled true")
+
+    # configure/set up the kitchen
+    kitchen = c['options'].get('kitchen')
+    kitchen_path = c['options'].get('kitchen_path', os.path.join(WORKING_DIR,
+                                                                 "kitchen"))
+    cookbook_path = os.path.join(kitchen_path, "cookbooks")
+    role_path = os.path.join(kitchen_path, "roles")
+    environment_path = os.path.join(kitchen_path, "environments")
+    client_config += "\ncookbook_path '%s'" % cookbook_path
+    client_config += "\nrole_path '%s'" % role_path
+    client_config += "\nenvironment_path '%s'" % environment_path
+    if kitchen:
+        log.debug("Cloning kitchen from %s", kitchen)
+        # remove the existing kitchen on update so we get a fresh clone
+        dep_action = next((input['value'] for input in c['inputs']
+                           if input['name'] == "deploy_action"), None)
+        if dep_action == "UPDATE":
+            shutil.rmtree(kitchen_path, ignore_errors=True)
+        cmd = ["git", "clone", kitchen, kitchen_path]
+        ret, out, err = run_subproc(cmd)
+        if ret != 0:
+            log.error("Error cloning kitchen from %s into %s: %s", kitchen,
+                      kitchen_path, err)
+            json.dump({'deploy_status_code': ret,
+                       'deploy_stdout': out,
+                       'deploy_stderr': err},
+                      sys.stdout)
+            return 0
+
+    # write the json attributes
+    ret, out, err = run_subproc(['hostname', '-f'])
+    if ret == 0:
+        fqdn = out.strip()
+    else:
+        err = "Could not determine hostname with hostname -f"
+        json.dump({'deploy_status_code': ret,
+                   'deploy_stdout': "",
+                   'deploy_stderr': err}, sys.stdout)
+        return 0
+    node_config = {}
+    for input in c['inputs']:
+        if input['name'] == 'environment':
+            client_config += "\nenvironment '%s'" % input['value']
+        elif input['name'] not in DEPLOY_KEYS:
+            node_config.update({input['name']: input['value']})
+    node_config.update({"run_list": json.loads(c['config'])})
+    node_path = os.path.join(WORKING_DIR, "node")
+    prepare_dir(node_path)
+    node_file = os.path.join(node_path, "%s.json" % fqdn)
+    with os.fdopen(os.open(node_file, os.O_CREAT | os.O_WRONLY, 0o600),
+                   'w') as f:
+        f.write(json.dumps(node_config, indent=4))
+    client_config += "\nnode_path '%s'" % node_path
+
+    # write out the completed client config
+    config_path = os.path.join(WORKING_DIR, "client.rb")
+    with os.fdopen(os.open(config_path, os.O_CREAT | os.O_WRONLY, 0o600),
+                   'w') as f:
+        f.write(client_config)
+
+    # run chef
+    heat_outputs_path = os.path.join(OUTPUTS_DIR, c['id'])
+    cmd = ['chef-client', '-z', '--config', config_path, "-j", node_file]
+    ret, out, err = run_subproc(cmd, heat_outputs_path=heat_outputs_path)
+    resp = {'deploy_status_code': ret,
+            'deploy_stdout': out,
+            'deploy_stderr': err}
+    log.debug("Chef output: %s", out)
+    if err:
+        log.error("Chef return code %s:\n%s", ret, err)
+    for output in c.get('outputs', []):
+        output_name = output['name']
+        try:
+            with open('%s.%s' % (heat_outputs_path, output_name)) as out:
+                resp[output_name] = out.read()
+        except IOError:
+            pass
+    json.dump(resp, sys.stdout)
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv))

hot/software-config/example-templates/example-chef-template.yaml

@@ -0,0 +1,76 @@
+heat_template_version: 2014-10-16
+
+parameters:
+
+  key_name:
+    type: string
+
+  flavor:
+    type: string
+    default: m1.small
+
+  image:
+    type: string
+    default: fedora-software-config
+
+resources:
+
+  the_sg:
+    type: OS::Neutron::SecurityGroup
+    properties:
+      name: the_sg
+      description: Ping and SSH
+      rules:
+      - protocol: icmp
+      - protocol: tcp
+        port_range_min: 22
+        port_range_max: 22
+
+  config:
+    type: OS::Heat::StructuredConfig
+    properties:
+      group: chef
+      inputs:
+      - name: nginx-pkg
+        type: Json
+      config:
+      - "recipe[nginx-pkg]"
+      options:
+        kitchen: "https://github.com/st-isidore-de-seville/cookbook-nginx-pkg.git"
+        kitchen_path: "/opt/heat/chef"
+
+  deployment:
+    type: OS::Heat::StructuredDeployment
+    properties:
+      config:
+        get_resource: structured_config
+      server:
+        get_resource: server
+      input_values:
+        nginx-pkg:
+          package:
+            name: nginx
+
+  server:
+    type: OS::Nova::Server
+    properties:
+      image: {get_param: image}
+      flavor: {get_param: flavor}
+      key_name: {get_param: key_name}
+      security_groups:
+      - {get_resource: the_sg}
+      user_data_format: SOFTWARE_CONFIG
+
+outputs:
+
+  status_code_deployment:
+    value:
+      get_attr: [deployment, deploy_status_code]
+
+  stdout:
+    value:
+      get_attr: [deployment, deploy_stdout]
+
+  stderr:
+    value:
+      get_attr: [deployment, deploy_stderr]

tests/software_config/test_hook_chef.py

@@ -0,0 +1,207 @@
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import copy
+import imp
+import json
+import logging
+import mock
+import StringIO
+import sys
+
+from tests.software_config import common
+
+log = logging.getLogger('test_hook_chef')
+
+
+@mock.patch("os.chdir")
+@mock.patch("os.makedirs")
+@mock.patch('subprocess.Popen')
+class HookChefTest(common.RunScriptTest):
+
+    data = {
+        'id': 'fake_stack',
+        'name': 'fake_resource_name',
+        'group': 'chef',
+        'inputs': [
+            {'name': 'fooval', 'value': {'bar': 'baz'}},
+            {'name': 'barval', 'value': {'foo': 'biff'}},
+            {'name': "deploy_server_id", 'value': 'foo'},
+            {'name': "deploy_action", 'value': 'foo'},
+            {'name': "deploy_stack_id", 'value': 'foo'},
+            {'name': "deploy_resource_name", 'value': 'foo'},
+            {'name': "deploy_signal_transport", 'value': 'foo'},
+            {'name': "deploy_signal_id", 'value': 'foo'},
+            {'name': "deploy_signal_verb", 'value': 'foo'}
+        ],
+        'options': {},
+        'outputs': [
+            {'name': 'first_output'},
+            {'name': 'second_output'}
+        ],
+        'config': None
+    }
+
+    def setUp(self):
+        super(HookChefTest, self).setUp()
+        self.hook_path = self.relative_path(
+            __file__,
+            '../..',
+            'hot/software-config/elements',
+            'heat-config-chef/install.d/hook-chef.py')
+        sys.stdin = StringIO.StringIO()
+        sys.stdout = StringIO.StringIO()
+
+    def tearDown(self):
+        super(HookChefTest, self).tearDown()
+        sys.stdin = sys.__stdin__
+        sys.stdout = sys.__stdout__
+
+    def get_module(self):
+        try:
+            imp.acquire_lock()
+            return imp.load_source("hook_chef", self.hook_path)
+        finally:
+            imp.release_lock()
+
+    def test_hook(self, mock_popen, mock_mkdirs, mock_chdir):
+        data = copy.deepcopy(self.data)
+        data['config'] = '["recipe[apache]"]'
+        hook_chef = self.get_module()
+        sys.stdin.write(json.dumps(data))
+        sys.stdin.seek(0)
+        mock_subproc = mock.Mock()
+        mock_popen.return_value = mock_subproc
+        mock_subproc.communicate.return_value = ("out", "err")
+        mock_subproc.returncode = 0
+        with mock.patch("os.fdopen", mock.mock_open()) as mfdopen:
+            with mock.patch("os.open", mock.mock_open()):
+                hook_chef.main(json.dumps(data))
+                exp_node = {
+                    'barval': {'foo': 'biff'},
+                    'fooval': {u'bar': u'baz'},
+                    'run_list': [u'recipe[apache]']
+                }
+                exp_node = json.dumps(exp_node, indent=4)
+                exp_cfg = ("log_level :debug\n"
+                           "log_location STDOUT\n"
+                           "local_mode true\n"
+                           "chef_zero.enabled true\n"
+                           "cookbook_path '/var/lib/heat-config/"
+                           "heat-config-chef/kitchen/cookbooks'\n"
+                           "role_path '/var/lib/heat-config/"
+                           "heat-config-chef/kitchen/roles'\n"
+                           "environment_path '/var/lib/heat-config/"
+                           "heat-config-chef/kitchen/environments'\n"
+                           "node_path '/var/lib/heat-config/"
+                           "heat-config-chef/node'")
+                mfdopen.return_value.write.assert_any_call(exp_cfg)
+                mfdopen.return_value.write.assert_any_call(exp_node)
+        calls = [
+            mock.call(['hostname', '-f'], env=mock.ANY, stderr=mock.ANY,
+                      stdout=mock.ANY),
+            mock.call([
+                'chef-client', '-z', '--config',
+                '/var/lib/heat-config/heat-config-chef/client.rb', '-j',
+                '/var/lib/heat-config/heat-config-chef/node/out.json'],
+                env=mock.ANY, stderr=mock.ANY, stdout=mock.ANY)
+        ]
+        mock_popen.assert_has_calls(calls, any_order=True)
+        self.assertEqual({"deploy_status_code": 0,
+                          "deploy_stdout": "out",
+                          "deploy_stderr": "err"},
+                         json.loads(sys.stdout.getvalue()))
+
+    def test_hook_with_kitchen(self, mock_popen, mock_mkdirs, mock_chdir):
+        data = copy.deepcopy(self.data)
+        data['config'] = '["recipe[apache]"]'
+        data['options'] = {
+            "kitchen": "https://github.com/fake.git",
+            "kitchen_path": "/opt/heat/chef/kitchen"
+        }
+        sys.stdin.write(json.dumps(data))
+        hook_chef = self.get_module()
+        sys.stdin.seek(0)
+        mock_subproc = mock.Mock()
+        mock_popen.return_value = mock_subproc
+        mock_subproc.communicate.return_value = ("out", "err")
+        mock_subproc.returncode = 0
+        with mock.patch("os.fdopen", mock.mock_open()) as mfdopen:
+            with mock.patch("os.open", mock.mock_open()):
+                hook_chef.main(json.dumps(data))
+                exp_cfg = ("log_level :debug\n"
+                           "log_location STDOUT\n"
+                           "local_mode true\n"
+                           "chef_zero.enabled true\n"
+                           "cookbook_path '/opt/heat/chef/kitchen/"
+                           "cookbooks'\n"
+                           "role_path '/opt/heat/chef/kitchen/roles'\n"
+                           "environment_path '/opt/heat/chef/kitchen/"
+                           "environments'\n"
+                           "node_path '/var/lib/heat-config/heat-config-chef"
+                           "/node'")
+                mfdopen.return_value.write.assert_any_call(exp_cfg)
+        calls = [
+            mock.call(['git', 'clone', "https://github.com/fake.git",
+                       "/opt/heat/chef/kitchen"], env=mock.ANY,
+                      stderr=mock.ANY, stdout=mock.ANY),
+            mock.call(['hostname', '-f'], env=mock.ANY, stderr=mock.ANY,
+                      stdout=mock.ANY),
+            mock.call([
+                'chef-client', '-z', '--config',
+                '/var/lib/heat-config/heat-config-chef/client.rb', '-j',
+                '/var/lib/heat-config/heat-config-chef/node/out.json'],
+                env=mock.ANY, stderr=mock.ANY, stdout=mock.ANY)
+        ]
+        mock_popen.assert_has_calls(calls, any_order=True)
+        self.assertEqual({"deploy_status_code": 0,
+                          "deploy_stdout": "out",
+                          "deploy_stderr": "err"},
+                         json.loads(sys.stdout.getvalue()))
+
+    def test_hook_environment(self, mock_popen, mock_mkdirs, mock_chdir):
+        data = copy.deepcopy(self.data)
+        data['config'] = '["recipe[apache]"]'
+        data['inputs'].append({'name': 'environment',
+                               'value': 'production'})
+        hook_chef = self.get_module()
+        sys.stdin.write(json.dumps(data))
+        sys.stdin.seek(0)
+        mock_subproc = mock.Mock()
+        mock_popen.return_value = mock_subproc
+        mock_subproc.communicate.return_value = ("out", "err")
+        mock_subproc.returncode = 0
+        with mock.patch("os.fdopen", mock.mock_open()) as mfdopen:
+            with mock.patch("os.open", mock.mock_open()):
+                hook_chef.main(json.dumps(data))
+                exp_node = {
+                    'barval': {'foo': 'biff'},
+                    'fooval': {u'bar': u'baz'},
+                    'run_list': [u'recipe[apache]']
+                }
+                exp_node = json.dumps(exp_node, indent=4)
+                exp_cfg = ("log_level :debug\n"
+                           "log_location STDOUT\n"
+                           "local_mode true\n"
+                           "chef_zero.enabled true\n"
+                           "cookbook_path '/var/lib/heat-config/"
+                           "heat-config-chef/kitchen/cookbooks'\n"
+                           "role_path '/var/lib/heat-config/"
+                           "heat-config-chef/kitchen/roles'\n"
+                           "environment_path '/var/lib/heat-config/"
+                           "heat-config-chef/kitchen/environments'\n"
+                           "environment 'production'\n"
+                           "node_path '/var/lib/heat-config/"
+                           "heat-config-chef/node'")
+                mfdopen.return_value.write.assert_any_call(exp_cfg)
+                mfdopen.return_value.write.assert_any_call(exp_node)