heat/heat/tests/engine/test_software_config.py

#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import datetime
import uuid

import mock
from oslo_messaging.rpc import dispatcher
from oslo_serialization import jsonutils as json
from oslo_utils import timeutils
import six

from heat.common import exception
from heat.common import template_format
from heat.engine.clients.os import swift
from heat.engine import service
from heat.engine import service_software_config
from heat.objects import resource as resource_objects
from heat.objects import software_deployment as software_deployment_object
from heat.tests import common
from heat.tests.engine import tools
from heat.tests import utils


class SoftwareConfigServiceTest(common.HeatTestCase):

    def setUp(self):
        super(SoftwareConfigServiceTest, self).setUp()
        self.ctx = utils.dummy_context()
        self.engine = service.EngineService('a-host', 'a-topic')

    def _create_software_config(
            self, group='Heat::Shell', name='config_mysql', config=None,
            inputs=None, outputs=None, options=None):
        inputs = inputs or []
        outputs = outputs or []
        options = options or {}
        return self.engine.create_software_config(
            self.ctx, group, name, config, inputs, outputs, options)

    def test_show_software_config(self):
        config_id = str(uuid.uuid4())

        ex = self.assertRaises(dispatcher.ExpectedException,
                               self.engine.show_software_config,
                               self.ctx, config_id)
        self.assertEqual(exception.NotFound, ex.exc_info[0])

        config = self._create_software_config()
        res = self.engine.show_software_config(self.ctx, config['id'])
        self.assertEqual(config, res)

    def test_create_software_config_new_ids(self):
        config1 = self._create_software_config()
        self.assertIsNotNone(config1)

        config2 = self._create_software_config()
        self.assertNotEqual(config1['id'], config2['id'])

    def test_create_software_config(self):
        kwargs = {
            'group': 'Heat::Chef',
            'name': 'config_heat',
            'config': '...',
            'inputs': [{'name': 'mode'}],
            'outputs': [{'name': 'endpoint'}],
            'options': {}
        }
        config = self._create_software_config(**kwargs)
        config_id = config['id']
        config = self.engine.show_software_config(self.ctx, config_id)
        self.assertEqual(kwargs['group'], config['group'])
        self.assertEqual(kwargs['name'], config['name'])
        self.assertEqual(kwargs['config'], config['config'])
        self.assertEqual(kwargs['inputs'], config['inputs'])
        self.assertEqual(kwargs['outputs'], config['outputs'])
        self.assertEqual(kwargs['options'], config['options'])

    def test_delete_software_config(self):
        config = self._create_software_config()
        self.assertIsNotNone(config)
        config_id = config['id']
        self.engine.delete_software_config(self.ctx, config_id)

        ex = self.assertRaises(dispatcher.ExpectedException,
                               self.engine.show_software_config,
                               self.ctx, config_id)
        self.assertEqual(exception.NotFound, ex.exc_info[0])

    def _create_software_deployment(self, config_id=None, input_values=None,
                                    action='INIT',
                                    status='COMPLETE', status_reason='',
                                    config_group=None,
                                    server_id=str(uuid.uuid4()),
                                    config_name=None,
                                    stack_user_project_id=None):
        input_values = input_values or {}
        if config_id is None:
            config = self._create_software_config(group=config_group,
                                                  name=config_name)
            config_id = config['id']
        return self.engine.create_software_deployment(
            self.ctx, server_id, config_id, input_values,
            action, status, status_reason, stack_user_project_id)

    def test_list_software_deployments(self):
        stack_name = 'test_list_software_deployments'
        t = template_format.parse(tools.wp_template)
        stack = utils.parse_stack(t, stack_name=stack_name)

        tools.setup_mocks(self.m, stack)
        self.m.ReplayAll()
        stack.store()
        stack.create()
        server = stack['WebServer']
        server_id = server.resource_id
        deployment = self._create_software_deployment(
            server_id=server_id)
        deployment_id = deployment['id']
        self.assertIsNotNone(deployment)

        deployments = self.engine.list_software_deployments(
            self.ctx, server_id=None)
        self.assertIsNotNone(deployments)
        deployment_ids = [x['id'] for x in deployments]
        self.assertIn(deployment_id, deployment_ids)
        self.assertIn(deployment, deployments)

        deployments = self.engine.list_software_deployments(
            self.ctx, server_id=str(uuid.uuid4()))
        self.assertEqual([], deployments)

        deployments = self.engine.list_software_deployments(
            self.ctx, server_id=server.resource_id)
        self.assertEqual([deployment], deployments)

        rs = resource_objects.Resource.get_by_physical_resource_id(
            self.ctx, server_id)
        self.assertEqual(deployment['config_id'],
                         rs.rsrc_metadata.get('deployments')[0]['id'])

    def test_metadata_software_deployments(self):
        stack_name = 'test_metadata_software_deployments'
        t = template_format.parse(tools.wp_template)
        stack = utils.parse_stack(t, stack_name=stack_name)

        tools.setup_mocks(self.m, stack)
        self.m.ReplayAll()
        stack.store()
        stack.create()
        server = stack['WebServer']
        server_id = server.resource_id

        stack_user_project_id = str(uuid.uuid4())
        d1 = self._create_software_deployment(
            config_group='mygroup',
            server_id=server_id,
            config_name='02_second',
            stack_user_project_id=stack_user_project_id)
        d2 = self._create_software_deployment(
            config_group='mygroup',
            server_id=server_id,
            config_name='01_first',
            stack_user_project_id=stack_user_project_id)
        d3 = self._create_software_deployment(
            config_group='myothergroup',
            server_id=server_id,
            config_name='03_third',
            stack_user_project_id=stack_user_project_id)
        metadata = self.engine.metadata_software_deployments(
            self.ctx, server_id=server_id)
        self.assertEqual(3, len(metadata))
        self.assertEqual('mygroup', metadata[1]['group'])
        self.assertEqual('mygroup', metadata[0]['group'])
        self.assertEqual('myothergroup', metadata[2]['group'])
        self.assertEqual(d1['config_id'], metadata[1]['id'])
        self.assertEqual(d2['config_id'], metadata[0]['id'])
        self.assertEqual(d3['config_id'], metadata[2]['id'])
        self.assertEqual('01_first', metadata[0]['name'])
        self.assertEqual('02_second', metadata[1]['name'])
        self.assertEqual('03_third', metadata[2]['name'])

        # assert that metadata via metadata_software_deployments matches
        # metadata via server resource
        rs = resource_objects.Resource.get_by_physical_resource_id(
            self.ctx, server_id)
        self.assertEqual(metadata, rs.rsrc_metadata.get('deployments'))

        deployments = self.engine.metadata_software_deployments(
            self.ctx, server_id=str(uuid.uuid4()))
        self.assertEqual([], deployments)

        # assert get results when the context tenant_id matches
        # the stored stack_user_project_id
        ctx = utils.dummy_context(tenant_id=stack_user_project_id)
        metadata = self.engine.metadata_software_deployments(
            ctx, server_id=server_id)
        self.assertEqual(3, len(metadata))

        # assert get no results when the context tenant_id is unknown
        ctx = utils.dummy_context(tenant_id=str(uuid.uuid4()))
        metadata = self.engine.metadata_software_deployments(
            ctx, server_id=server_id)
        self.assertEqual(0, len(metadata))

    def test_show_software_deployment(self):
        deployment_id = str(uuid.uuid4())
        ex = self.assertRaises(dispatcher.ExpectedException,
                               self.engine.show_software_deployment,
                               self.ctx, deployment_id)
        self.assertEqual(exception.NotFound, ex.exc_info[0])

        deployment = self._create_software_deployment()
        self.assertIsNotNone(deployment)
        deployment_id = deployment['id']
        self.assertEqual(
            deployment,
            self.engine.show_software_deployment(self.ctx, deployment_id))

    @mock.patch.object(service_software_config.SoftwareConfigService,
                       '_push_metadata_software_deployments')
    def test_signal_software_deployment(self, pmsd):
        self.assertRaises(ValueError,
                          self.engine.signal_software_deployment,
                          self.ctx, None, {}, None)
        deployment_id = str(uuid.uuid4())
        ex = self.assertRaises(dispatcher.ExpectedException,
                               self.engine.signal_software_deployment,
                               self.ctx, deployment_id, {}, None)
        self.assertEqual(exception.NotFound, ex.exc_info[0])

        deployment = self._create_software_deployment()
        deployment_id = deployment['id']

        # signal is ignore unless deployment is IN_PROGRESS
        self.assertIsNone(self.engine.signal_software_deployment(
            self.ctx, deployment_id, {}, None))

        # simple signal, no data
        deployment = self._create_software_deployment(action='INIT',
                                                      status='IN_PROGRESS')
        deployment_id = deployment['id']
        res = self.engine.signal_software_deployment(
            self.ctx, deployment_id, {}, None)
        self.assertEqual('deployment succeeded', res)

        sd = software_deployment_object.SoftwareDeployment.get_by_id(
            self.ctx, deployment_id)
        self.assertEqual('COMPLETE', sd.status)
        self.assertEqual('Outputs received', sd.status_reason)
        self.assertEqual({
            'deploy_status_code': None,
            'deploy_stderr': None,
            'deploy_stdout': None
        }, sd.output_values)
        self.assertIsNotNone(sd.updated_at)

        # simple signal, some data
        config = self._create_software_config(outputs=[{'name': 'foo'}])
        deployment = self._create_software_deployment(
            config_id=config['id'], action='INIT', status='IN_PROGRESS')
        deployment_id = deployment['id']
        result = self.engine.signal_software_deployment(
            self.ctx,
            deployment_id,
            {'foo': 'bar', 'deploy_status_code': 0},
            None)
        self.assertEqual('deployment succeeded', result)
        sd = software_deployment_object.SoftwareDeployment.get_by_id(
            self.ctx, deployment_id)
        self.assertEqual('COMPLETE', sd.status)
        self.assertEqual('Outputs received', sd.status_reason)
        self.assertEqual({
            'deploy_status_code': 0,
            'foo': 'bar',
            'deploy_stderr': None,
            'deploy_stdout': None
        }, sd.output_values)
        self.assertIsNotNone(sd.updated_at)

        # failed signal on deploy_status_code
        config = self._create_software_config(outputs=[{'name': 'foo'}])
        deployment = self._create_software_deployment(
            config_id=config['id'], action='INIT', status='IN_PROGRESS')
        deployment_id = deployment['id']
        result = self.engine.signal_software_deployment(
            self.ctx,
            deployment_id,
            {
                'foo': 'bar',
                'deploy_status_code': -1,
                'deploy_stderr': 'Its gone Pete Tong'
            },
            None)
        self.assertEqual('deployment failed (-1)', result)
        sd = software_deployment_object.SoftwareDeployment.get_by_id(
            self.ctx, deployment_id)
        self.assertEqual('FAILED', sd.status)
        self.assertEqual(
            ('deploy_status_code : Deployment exited with non-zero '
             'status code: -1'),
            sd.status_reason)
        self.assertEqual({
            'deploy_status_code': -1,
            'foo': 'bar',
            'deploy_stderr': 'Its gone Pete Tong',
            'deploy_stdout': None
        }, sd.output_values)
        self.assertIsNotNone(sd.updated_at)

        # failed signal on error_output foo
        config = self._create_software_config(outputs=[
            {'name': 'foo', 'error_output': True}])
        deployment = self._create_software_deployment(
            config_id=config['id'], action='INIT', status='IN_PROGRESS')
        deployment_id = deployment['id']
        result = self.engine.signal_software_deployment(
            self.ctx,
            deployment_id,
            {
                'foo': 'bar',
                'deploy_status_code': -1,
                'deploy_stderr': 'Its gone Pete Tong'
            },
            None)
        self.assertEqual('deployment failed', result)

        sd = software_deployment_object.SoftwareDeployment.get_by_id(
            self.ctx, deployment_id)
        self.assertEqual('FAILED', sd.status)
        self.assertEqual(
            ('foo : bar, deploy_status_code : Deployment exited with '
             'non-zero status code: -1'),
            sd.status_reason)
        self.assertEqual({
            'deploy_status_code': -1,
            'foo': 'bar',
            'deploy_stderr': 'Its gone Pete Tong',
            'deploy_stdout': None
        }, sd.output_values)
        self.assertIsNotNone(sd.updated_at)

    def test_create_software_deployment(self):
        kwargs = {
            'group': 'Heat::Chef',
            'name': 'config_heat',
            'config': '...',
            'inputs': [{'name': 'mode'}],
            'outputs': [{'name': 'endpoint'}],
            'options': {}
        }
        config = self._create_software_config(**kwargs)
        config_id = config['id']
        kwargs = {
            'config_id': config_id,
            'input_values': {'mode': 'standalone'},
            'action': 'INIT',
            'status': 'COMPLETE',
            'status_reason': ''
        }
        deployment = self._create_software_deployment(**kwargs)
        deployment_id = deployment['id']
        deployment = self.engine.show_software_deployment(
            self.ctx, deployment_id)
        self.assertEqual(deployment_id, deployment['id'])
        self.assertEqual(kwargs['input_values'], deployment['input_values'])

    @mock.patch.object(service_software_config.SoftwareConfigService,
                       '_refresh_software_deployment')
    def test_show_software_deployment_refresh(
            self, _refresh_software_deployment):
        temp_url = ('http://192.0.2.1/v1/AUTH_a/b/c'
                    '?temp_url_sig=ctemp_url_expires=1234')
        config = self._create_software_config(inputs=[
            {
                'name': 'deploy_signal_transport',
                'type': 'String',
                'value': 'TEMP_URL_SIGNAL'
            }, {
                'name': 'deploy_signal_id',
                'type': 'String',
                'value': temp_url
            }
        ])

        deployment = self._create_software_deployment(
            status='IN_PROGRESS', config_id=config['id'])

        deployment_id = deployment['id']
        sd = software_deployment_object.SoftwareDeployment.get_by_id(
            self.ctx, deployment_id)
        _refresh_software_deployment.return_value = sd
        self.assertEqual(
            deployment,
            self.engine.show_software_deployment(self.ctx, deployment_id))
        self.assertEqual(
            (self.ctx, sd, temp_url),
            _refresh_software_deployment.call_args[0])

    def test_update_software_deployment_new_config(self):

        server_id = str(uuid.uuid4())
        mock_push = self.patchobject(self.engine.software_config,
                                     '_push_metadata_software_deployments')

        deployment = self._create_software_deployment(server_id=server_id)
        self.assertIsNotNone(deployment)
        deployment_id = deployment['id']
        deployment_action = deployment['action']
        self.assertEqual('INIT', deployment_action)
        config_id = deployment['config_id']
        self.assertIsNotNone(config_id)
        updated = self.engine.update_software_deployment(
            self.ctx, deployment_id=deployment_id, config_id=config_id,
            input_values={}, output_values={}, action='DEPLOY',
            status='WAITING', status_reason='', updated_at=None)
        self.assertIsNotNone(updated)
        self.assertEqual(config_id, updated['config_id'])
        self.assertEqual('DEPLOY', updated['action'])
        self.assertEqual('WAITING', updated['status'])
        self.assertEqual(2, mock_push.call_count)

    def test_update_software_deployment_status(self):

        server_id = str(uuid.uuid4())
        mock_push = self.patchobject(self.engine.software_config,
                                     '_push_metadata_software_deployments')

        deployment = self._create_software_deployment(server_id=server_id)

        self.assertIsNotNone(deployment)
        deployment_id = deployment['id']
        deployment_action = deployment['action']
        self.assertEqual('INIT', deployment_action)
        updated = self.engine.update_software_deployment(
            self.ctx, deployment_id=deployment_id, config_id=None,
            input_values=None, output_values={}, action='DEPLOY',
            status='WAITING', status_reason='', updated_at=None)
        self.assertIsNotNone(updated)
        self.assertEqual('DEPLOY', updated['action'])
        self.assertEqual('WAITING', updated['status'])
        mock_push.assert_called_once_with(self.ctx, server_id)

    def test_update_software_deployment_fields(self):

        deployment = self._create_software_deployment()
        deployment_id = deployment['id']
        config_id = deployment['config_id']

        def check_software_deployment_updated(**kwargs):
            values = {
                'config_id': None,
                'input_values': {},
                'output_values': {},
                'action': {},
                'status': 'WAITING',
                'status_reason': ''
            }
            values.update(kwargs)
            updated = self.engine.update_software_deployment(
                self.ctx, deployment_id, updated_at=None, **values)
            for key, value in six.iteritems(kwargs):
                self.assertEqual(value, updated[key])

        check_software_deployment_updated(config_id=config_id)
        check_software_deployment_updated(input_values={'foo': 'fooooo'})
        check_software_deployment_updated(output_values={'bar': 'baaaaa'})
        check_software_deployment_updated(action='DEPLOY')
        check_software_deployment_updated(status='COMPLETE')
        check_software_deployment_updated(status_reason='Done!')

    def test_delete_software_deployment(self):
        deployment_id = str(uuid.uuid4())
        ex = self.assertRaises(dispatcher.ExpectedException,
                               self.engine.delete_software_deployment,
                               self.ctx, deployment_id)
        self.assertEqual(exception.NotFound, ex.exc_info[0])

        deployment = self._create_software_deployment()
        self.assertIsNotNone(deployment)
        deployment_id = deployment['id']
        deployments = self.engine.list_software_deployments(
            self.ctx, server_id=None)
        deployment_ids = [x['id'] for x in deployments]
        self.assertIn(deployment_id, deployment_ids)
        self.engine.delete_software_deployment(self.ctx, deployment_id)
        deployments = self.engine.list_software_deployments(
            self.ctx, server_id=None)
        deployment_ids = [x['id'] for x in deployments]
        self.assertNotIn(deployment_id, deployment_ids)

    @mock.patch.object(service_software_config.SoftwareConfigService,
                       'metadata_software_deployments')
    @mock.patch.object(service_software_config.resource_object.Resource,
                       'get_by_physical_resource_id')
    @mock.patch.object(service_software_config.requests, 'put')
    def test_push_metadata_software_deployments(self, put, res_get, md_sd):
        rs = mock.Mock()
        rs.rsrc_metadata = {'original': 'metadata'}
        rs.data = []
        res_get.return_value = rs

        deployments = {'deploy': 'this'}
        md_sd.return_value = deployments

        result_metadata = {
            'original': 'metadata',
            'deployments': {'deploy': 'this'}
        }

        self.engine.software_config._push_metadata_software_deployments(
            self.ctx, '1234')
        rs.update_and_save.assert_called_once_with(
            {'rsrc_metadata': result_metadata})
        put.side_effect = Exception('Unexpected requests.put')

    @mock.patch.object(service_software_config.SoftwareConfigService,
                       'metadata_software_deployments')
    @mock.patch.object(service_software_config.resource_object.Resource,
                       'get_by_physical_resource_id')
    @mock.patch.object(service_software_config.requests, 'put')
    def test_push_metadata_software_deployments_temp_url(
            self, put, res_get, md_sd):
        rs = mock.Mock()
        rs.rsrc_metadata = {'original': 'metadata'}
        rd = mock.Mock()
        rd.key = 'metadata_put_url'
        rd.value = 'http://192.168.2.2/foo/bar'
        rs.data = [rd]
        res_get.return_value = rs

        deployments = {'deploy': 'this'}
        md_sd.return_value = deployments

        result_metadata = {
            'original': 'metadata',
            'deployments': {'deploy': 'this'}
        }

        self.engine.software_config._push_metadata_software_deployments(
            self.ctx, '1234')
        rs.update_and_save.assert_called_once_with(
            {'rsrc_metadata': result_metadata})

        put.assert_called_once_with(
            'http://192.168.2.2/foo/bar', json.dumps(result_metadata))

    @mock.patch.object(service_software_config.SoftwareConfigService,
                       'signal_software_deployment')
    @mock.patch.object(swift.SwiftClientPlugin, '_create')
    def test_refresh_software_deployment(self, scc, ssd):
        temp_url = ('http://192.0.2.1/v1/AUTH_a/b/c'
                    '?temp_url_sig=ctemp_url_expires=1234')
        container = 'b'
        object_name = 'c'

        config = self._create_software_config(inputs=[
            {
                'name': 'deploy_signal_transport',
                'type': 'String',
                'value': 'TEMP_URL_SIGNAL'
            }, {
                'name': 'deploy_signal_id',
                'type': 'String',
                'value': temp_url
            }
        ])

        timeutils.set_time_override(
            datetime.datetime(2013, 1, 23, 22, 48, 5, 0))
        self.addCleanup(timeutils.clear_time_override)
        now = timeutils.utcnow()
        then = now - datetime.timedelta(0, 60)

        last_modified_1 = 'Wed, 23 Jan 2013 22:47:05 GMT'
        last_modified_2 = 'Wed, 23 Jan 2013 22:48:05 GMT'

        sc = mock.MagicMock()
        headers = {
            'last-modified': last_modified_1
        }
        sc.head_object.return_value = headers
        sc.get_object.return_value = (headers, '{"foo": "bar"}')
        scc.return_value = sc

        deployment = self._create_software_deployment(
            status='IN_PROGRESS', config_id=config['id'])

        deployment_id = six.text_type(deployment['id'])
        sd = software_deployment_object.SoftwareDeployment.get_by_id(
            self.ctx, deployment_id)

        # poll with missing object
        swift_exc = swift.SwiftClientPlugin.exceptions_module
        sc.head_object.side_effect = swift_exc.ClientException(
            'Not found', http_status=404)

        self.assertEqual(
            sd,
            self.engine.software_config._refresh_software_deployment(
                self.ctx, sd, temp_url))
        sc.head_object.assert_called_once_with(container, object_name)
        # no call to get_object or signal_last_modified
        self.assertEqual([], sc.get_object.mock_calls)
        self.assertEqual([], ssd.mock_calls)

        # poll with other error
        sc.head_object.side_effect = swift_exc.ClientException(
            'Ouch', http_status=409)
        self.assertRaises(
            swift_exc.ClientException,
            self.engine.software_config._refresh_software_deployment,
            self.ctx,
            sd,
            temp_url)
        # no call to get_object or signal_last_modified
        self.assertEqual([], sc.get_object.mock_calls)
        self.assertEqual([], ssd.mock_calls)
        sc.head_object.side_effect = None

        # first poll populates data signal_last_modified
        self.engine.software_config._refresh_software_deployment(
            self.ctx, sd, temp_url)
        sc.head_object.assert_called_with(container, object_name)
        sc.get_object.assert_called_once_with(container, object_name)
        # signal_software_deployment called with signal
        ssd.assert_called_once_with(self.ctx, deployment_id, {u"foo": u"bar"},
                                    timeutils.strtime(then))

        # second poll updated_at populated with first poll last-modified
        software_deployment_object.SoftwareDeployment.update_by_id(
            self.ctx, deployment_id, {'updated_at': then})
        sd = software_deployment_object.SoftwareDeployment.get_by_id(
            self.ctx, deployment_id)
        self.assertEqual(then, sd.updated_at)
        self.engine.software_config._refresh_software_deployment(
            self.ctx, sd, temp_url)
        sc.get_object.assert_called_once_with(container, object_name)
        # signal_software_deployment has not been called again
        ssd.assert_called_once_with(self.ctx, deployment_id, {"foo": "bar"},
                                    timeutils.strtime(then))

        # third poll last-modified changed, new signal
        headers['last-modified'] = last_modified_2
        sc.head_object.return_value = headers
        sc.get_object.return_value = (headers, '{"bar": "baz"}')
        self.engine.software_config._refresh_software_deployment(
            self.ctx, sd, temp_url)

        # two calls to signal_software_deployment, for then and now
        self.assertEqual(2, len(ssd.mock_calls))
        ssd.assert_called_with(self.ctx, deployment_id, {"bar": "baz"},
                               timeutils.strtime(now))

        # four polls result in only two signals, for then and now
        software_deployment_object.SoftwareDeployment.update_by_id(
            self.ctx, deployment_id, {'updated_at': now})
        sd = software_deployment_object.SoftwareDeployment.get_by_id(
            self.ctx, deployment_id)
        self.engine.software_config._refresh_software_deployment(
            self.ctx, sd, temp_url)
        self.assertEqual(2, len(ssd.mock_calls))