Break out signalling heat-config-notify command

Currently signalling heat is done in a chunk of python at the end
of 55-heat-config. This change moves the signalling to its own
command heat-config-notify, which can be called from 55-heat-config
or anything else which wants to signal heat.

This change also adds proper test coverage to the signalling logic.

Change-Id: I6ba7468b46ca8cfad1e58207cd3f814a178ff6f1

hot/software-config/elements/heat-config/bin/heat-config-notify

@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import json
+import logging
+import os
+import sys
+
+import requests
+
+try:
+    from heatclient import client as heatclient
+except ImportError:
+    heatclient = None
+
+try:
+    from keystoneclient.v3 import client as ksclient
+except ImportError:
+    ksclient = None
+
+
+def init_logging():
+    log = logging.getLogger('heat-config-notify')
+    handler = logging.StreamHandler(sys.stderr)
+    handler.setFormatter(
+        logging.Formatter(
+            '[%(asctime)s] (%(name)s) [%(levelname)s] %(message)s'))
+    log.addHandler(handler)
+    log.setLevel('DEBUG')
+
+
+def main(argv=sys.argv, stdin=sys.stdin):
+
+    log = init_logging()
+    usage = ('Usage:\n  heat-config-notify /path/to/config.json '
+             '< /path/to/signal_data.json')
+
+    if len(argv) < 2:
+        log.error(usage)
+        return 1
+
+    try:
+        signal_data = json.load(stdin)
+    except ValueError:
+        log.warn('No valid json found on stdin')
+        signal_data = {}
+
+    conf_file = argv[1]
+    if not os.path.exists(conf_file):
+        log.error('No config file %s' % conf_file)
+        log.error(usage)
+        return 1
+
+    c = json.load(open(conf_file))
+
+    iv = dict((i['name'], i['value']) for i in c['inputs'])
+
+    if 'deploy_signal_id' in iv:
+        sigurl = iv.get('deploy_signal_id')
+        signal_data = json.dumps(signal_data)
+        log.debug('Signalling to %s' % sigurl)
+        r = requests.post(sigurl, data=signal_data,
+                          headers={'content-type': None})
+        log.debug('Response %s ' % r)
+
+    if 'deploy_auth_url' in iv:
+        ks = ksclient.Client(
+            auth_url=iv['deploy_auth_url'],
+            user_id=iv['deploy_user_id'],
+            password=iv['deploy_password'],
+            project_id=iv['deploy_project_id'])
+        endpoint = ks.service_catalog.url_for(
+            service_type='orchestration', endpoint_type='publicURL')
+        log.debug('Signalling to %s' % endpoint)
+        heat = heatclient.Client(
+            '1', endpoint, token=ks.auth_token)
+        r = heat.resources.signal(
+            iv.get('deploy_stack_id'),
+            iv.get('deploy_resource_name'),
+            data=signal_data)
+        log.debug('Response %s ' % r)
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv, sys.stdin))

hot/software-config/elements/heat-config/os-refresh-config/configure.d/55-heat-config

@@ -26,6 +26,8 @@ CONF_FILE = os.environ.get('HEAT_SHELL_CONFIG',
                            '/var/run/heat-config/heat-config')
 DEPLOYED_DIR = os.environ.get('HEAT_CONFIG_DEPLOYED',
                               '/var/run/heat-config/deployed')
+HEAT_CONFIG_NOTIFY = os.environ.get('HEAT_CONFIG_NOTIFY',
+                                    'heat-config-notify')
 
 
 def main(argv=sys.argv):
@@ -100,71 +102,67 @@ def invoke_hook(c, log):
         x for x in c['group'] if x == '-' or x == '_' or x.isalnum())
     hook_path = os.path.join(HOOKS_DIR, hook)
 
-    signal_data = None
+    signal_data = {}
     if not os.path.exists(hook_path):
         log.warn('Skipping group %s with no hook script %s' % (
             c['group'], hook_path))
+        return
+
+    # write out config, which indicates it is deployed regardless of
+    # subsequent hook success
+    with os.fdopen(os.open(
+            deployed_path, os.O_CREAT | os.O_WRONLY, 0o600), 'w') as f:
+        json.dump(c, f, indent=2)
+
+    log.debug('Running %s < %s' % (hook_path, deployed_path))
+    subproc = subprocess.Popen([hook_path],
+                               stdin=subprocess.PIPE,
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE)
+    stdout, stderr = subproc.communicate(input=json.dumps(c))
+
+    log.info(stdout)
+    log.debug(stderr)
+
+    if subproc.returncode:
+        log.error("Error running %s. [%s]\n" % (
+            hook_path, subproc.returncode))
     else:
+        log.info('Completed %s' % hook_path)
 
-        # write out config, which indicates it is deployed regardless of
+    try:
-        # subsequent hook success
+        if stdout:
-        with os.fdopen(os.open(
+            signal_data = json.loads(stdout)
-                deployed_path, os.O_CREAT | os.O_WRONLY, 0o600), 'w') as f:
+    except ValueError:
-            json.dump(c, f, indent=2)
+        signal_data = {
+            'deploy_stdout': stdout,
+            'deploy_stderr': stderr,
+            'deploy_status_code': subproc.returncode,
+        }
 
-        log.debug('Running %s < %s' % (hook_path, deployed_path))
+    signal_data_path = os.path.join(DEPLOYED_DIR, '%s.notify.json' % c['id'])
-        subproc = subprocess.Popen([hook_path],
+    # write out notify data for debugging
-                                   stdin=subprocess.PIPE,
+    with os.fdopen(os.open(
-                                   stdout=subprocess.PIPE,
+            signal_data_path, os.O_CREAT | os.O_WRONLY, 0o600), 'w') as f:
-                                   stderr=subprocess.PIPE)
+        json.dump(signal_data, f, indent=2)
-        stdout, stderr = subproc.communicate(input=json.dumps(c))
 
-        log.info(stdout)
+    log.debug('Running %s %s < %s' % (
+        HEAT_CONFIG_NOTIFY, deployed_path, signal_data_path))
+    subproc = subprocess.Popen([HEAT_CONFIG_NOTIFY, deployed_path],
+                               stdin=subprocess.PIPE,
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE)
+    stdout, stderr = subproc.communicate(input=json.dumps(signal_data))
+
+    log.info(stdout)
+
+    if subproc.returncode:
+        log.error(
+            "Error running heat-config-notify. [%s]\n" % subproc.returncode)
+        log.error(stderr)
+    else:
         log.debug(stderr)
 
-        if subproc.returncode:
-            log.error("Error running %s. [%s]\n" % (
-                hook_path, subproc.returncode))
-        else:
-            log.info('Completed %s' % hook_path)
-
-        try:
-            if stdout:
-                signal_data = json.loads(stdout)
-        except ValueError:
-            signal_data = {
-                'deploy_stdout': stdout,
-                'deploy_stderr': stderr,
-                'deploy_status_code': subproc.returncode,
-            }
-
-    if signal_data:
-        if 'deploy_signal_id' in iv:
-            sigurl = iv.get('deploy_signal_id')
-            signal_data = json.dumps(signal_data)
-            log.debug('Signalling to %s with %s' % (sigurl, signal_data))
-            r = requests.post(sigurl, data=signal_data,
-                              headers={'content-type': None})
-            log.debug('Response %s ' % r)
-        if 'deploy_auth_url' in iv:
-            from heatclient import client as heatclient
-            from keystoneclient.v3 import client as ksclient
-            ks = ksclient.Client(
-                auth_url=iv['deploy_auth_url'],
-                user_id=iv['deploy_user_id'],
-                password=iv['deploy_password'],
-                project_id=iv['deploy_project_id'])
-            endpoint = ks.service_catalog.url_for(
-                service_type='orchestration', endpoint_type='publicURL')
-            log.debug('Signalling to %s' % endpoint)
-            heat = heatclient.Client(
-                '1', endpoint, token=ks.auth_token)
-            r = heat.resources.signal(
-                iv.get('deploy_stack_id'),
-                iv.get('deploy_resource_name'),
-                data=signal_data)
-            log.debug('Response %s ' % r)
-
 
 if __name__ == '__main__':
     sys.exit(main(sys.argv))

tests/software_config/heat_config_notify.py

@@ -0,0 +1 @@
+../../hot/software-config/elements/heat-config/bin/heat-config-notify

tests/software_config/test_heat_config.py

@@ -17,7 +17,6 @@ import os
 import tempfile
 
 import fixtures
-import requests_mock
 from testtools import matchers
 
 from tests.software_config import common
@@ -154,10 +153,7 @@ class HeatConfigTest(common.RunScriptTest):
             hook_path = self.hooks_dir.join(hook)
             self.assertThat(hook_path, matchers.FileExists())
 
-    @requests_mock.Mocker(kw='mock_request')
+    def test_run_heat_config(self):
-    def test_run_heat_config(self, mock_request):
-        mock_request.register_uri('POST', 'mock://192.0.2.2/foo')
-        mock_request.register_uri('POST', 'mock://192.0.2.3/foo')
 
         self.run_heat_config(self.data)
 

tests/software_config/test_heat_config_notify.py

@@ -0,0 +1,178 @@
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import cStringIO
+import json
+import tempfile
+
+import fixtures
+import mock
+
+from tests.software_config import common
+from tests.software_config import heat_config_notify as hcn
+
+
+class HeatConfigNotifyTest(common.RunScriptTest):
+
+    data_signal_id = {
+        'id': '5555',
+        'group': 'script',
+        'inputs': [{
+            'name': 'deploy_signal_id',
+            'value': 'mock://192.0.2.3/foo'
+        }],
+        'config': 'five'
+    }
+
+    data_heat_signal = {
+        'id': '5555',
+        'group': 'script',
+        'inputs': [{
+            'name': 'deploy_auth_url',
+            'value': 'mock://192.0.2.3/auth'
+        }, {
+            'name': 'deploy_user_id',
+            'value': 'aaaa'
+        }, {
+            'name': 'deploy_password',
+            'value': 'password'
+        }, {
+            'name': 'deploy_project_id',
+            'value': 'bbbb'
+        }, {
+            'name': 'deploy_stack_id',
+            'value': 'cccc'
+        }, {
+            'name': 'deploy_resource_name',
+            'value': 'the_resource'
+        }],
+        'config': 'five'
+    }
+
+    def setUp(self):
+        super(HeatConfigNotifyTest, self).setUp()
+        self.deployed_dir = self.useFixture(fixtures.TempDir())
+        hcn.init_logging = mock.MagicMock()
+
+    def write_config_file(self, data):
+        config_file = tempfile.NamedTemporaryFile()
+        config_file.write(json.dumps(data))
+        config_file.flush()
+        return config_file
+
+    def test_notify_missing_file(self):
+
+        signal_data = json.dumps({'foo': 'bar'})
+        stdin = cStringIO.StringIO(signal_data)
+
+        with self.write_config_file(self.data_signal_id) as config_file:
+            config_file_name = config_file.name
+
+        self.assertEqual(
+            1, hcn.main(['heat-config-notify', config_file_name], stdin))
+
+    def test_notify_missing_file_arg(self):
+
+        signal_data = json.dumps({'foo': 'bar'})
+        stdin = cStringIO.StringIO(signal_data)
+
+        self.assertEqual(
+            1, hcn.main(['heat-config-notify'], stdin))
+
+    def test_notify_signal_id(self):
+        requests = mock.MagicMock()
+        hcn.requests = requests
+
+        requests.post.return_value = '[200]'
+
+        signal_data = json.dumps({'foo': 'bar'})
+        stdin = cStringIO.StringIO(signal_data)
+
+        with self.write_config_file(self.data_signal_id) as config_file:
+            self.assertEqual(
+                0, hcn.main(['heat-config-notify', config_file.name], stdin))
+
+        requests.post.assert_called_once_with(
+            'mock://192.0.2.3/foo',
+            data=signal_data,
+            headers={'content-type': None})
+
+    def test_notify_signal_id_empty_data(self):
+        requests = mock.MagicMock()
+        hcn.requests = requests
+
+        requests.post.return_value = '[200]'
+
+        stdin = cStringIO.StringIO()
+
+        with self.write_config_file(self.data_signal_id) as config_file:
+            self.assertEqual(
+                0, hcn.main(['heat-config-notify', config_file.name], stdin))
+
+        requests.post.assert_called_once_with(
+            'mock://192.0.2.3/foo',
+            data='{}',
+            headers={'content-type': None})
+
+    def test_notify_signal_id_invalid_json_data(self):
+        requests = mock.MagicMock()
+        hcn.requests = requests
+
+        requests.post.return_value = '[200]'
+
+        stdin = cStringIO.StringIO('{{{"hi')
+
+        with self.write_config_file(self.data_signal_id) as config_file:
+            self.assertEqual(
+                0, hcn.main(['heat-config-notify', config_file.name], stdin))
+
+        requests.post.assert_called_once_with(
+            'mock://192.0.2.3/foo',
+            data='{}',
+            headers={'content-type': None})
+
+    def test_notify_heat_signal(self):
+        ksclient = mock.MagicMock()
+        hcn.ksclient = ksclient
+        ks = mock.MagicMock()
+        ksclient.Client.return_value = ks
+
+        heatclient = mock.MagicMock()
+        hcn.heatclient = heatclient
+        heat = mock.MagicMock()
+        heatclient.Client.return_value = heat
+
+        signal_data = json.dumps({'foo': 'bar'})
+        stdin = cStringIO.StringIO(signal_data)
+
+        ks.service_catalog.url_for.return_value = 'mock://192.0.2.3/heat'
+        heat.resources.signal.return_value = 'all good'
+
+        with self.write_config_file(self.data_heat_signal) as config_file:
+            self.assertEqual(
+                0, hcn.main(['heat-config-notify', config_file.name], stdin))
+
+        ksclient.Client.assert_called_once_with(
+            auth_url='mock://192.0.2.3/auth',
+            user_id='aaaa',
+            password='password',
+            project_id='bbbb')
+        ks.service_catalog.url_for.assert_called_once_with(
+            service_type='orchestration', endpoint_type='publicURL')
+
+        heatclient.Client.assert_called_once_with(
+            '1', 'mock://192.0.2.3/heat', token=ks.auth_token)
+        heat.resources.signal.assert_called_once_with(
+            'cccc',
+            'the_resource',
+            data={'foo': 'bar'})