diff --git a/heat/engine/clients/os/swift.py b/heat/engine/clients/os/swift.py index 42a29039d..38e90b881 100644 --- a/heat/engine/clients/os/swift.py +++ b/heat/engine/clients/os/swift.py @@ -11,6 +11,8 @@ # License for the specific language governing permissions and limitations # under the License. +import datetime +import email.utils import hashlib import random import time @@ -117,3 +119,19 @@ class SwiftClientPlugin(client_plugin.ClientPlugin): self.client().put_object(container_name, obj_name, IN_PROGRESS) return self.get_temp_url(container_name, obj_name, timeout) + + def parse_last_modified(self, lm): + ''' + Parses the last-modified value, such as from a swift object header, + and returns the datetime.datetime of that value. + + :param lm: The last-modified value (or None) + :type lm: string + :returns: An offset-naive UTC datetime of the value (or None) + ''' + if not lm: + return None + pd = email.utils.parsedate(lm)[:6] + # according to RFC 2616, all HTTP time headers must be + # in GMT time, so create an offset-naive UTC datetime + return datetime.datetime(*pd) diff --git a/heat/engine/resources/software_config/software_deployment.py b/heat/engine/resources/software_config/software_deployment.py index 8b2c258ec..d14323212 100644 --- a/heat/engine/resources/software_config/software_deployment.py +++ b/heat/engine/resources/software_config/software_deployment.py @@ -91,19 +91,21 @@ class SoftwareDeployment(signal_responder.SignalResponder): DEPLOY_SIGNAL_ID, DEPLOY_STACK_ID, DEPLOY_RESOURCE_NAME, DEPLOY_AUTH_URL, DEPLOY_USERNAME, DEPLOY_PASSWORD, - DEPLOY_PROJECT_ID, DEPLOY_USER_ID + DEPLOY_PROJECT_ID, DEPLOY_USER_ID, + DEPLOY_SIGNAL_VERB, DEPLOY_SIGNAL_TRANSPORT ) = ( 'deploy_server_id', 'deploy_action', 'deploy_signal_id', 'deploy_stack_id', 'deploy_resource_name', 'deploy_auth_url', 'deploy_username', 'deploy_password', - 'deploy_project_id', 'deploy_user_id' + 'deploy_project_id', 'deploy_user_id', + 'deploy_signal_verb', 'deploy_signal_transport' ) SIGNAL_TRANSPORTS = ( - CFN_SIGNAL, HEAT_SIGNAL, NO_SIGNAL + CFN_SIGNAL, TEMP_URL_SIGNAL, HEAT_SIGNAL, NO_SIGNAL ) = ( - 'CFN_SIGNAL', 'HEAT_SIGNAL', 'NO_SIGNAL' + 'CFN_SIGNAL', 'TEMP_URL_SIGNAL', 'HEAT_SIGNAL', 'NO_SIGNAL' ) properties_schema = { @@ -144,10 +146,12 @@ class SoftwareDeployment(signal_responder.SignalResponder): properties.Schema.STRING, _('How the server should signal to heat with the deployment ' 'output values. CFN_SIGNAL will allow an HTTP POST to a CFN ' - 'keypair signed URL. HEAT_SIGNAL will allow calls to ' - 'the Heat API resource-signal using the provided keystone ' - 'credentials. NO_SIGNAL will result in the resource going to ' - 'the COMPLETE state without waiting for any signal.'), + 'keypair signed URL. TEMP_URL_SIGNAL will create a ' + 'Swift TempURL to be signaled via HTTP PUT. HEAT_SIGNAL ' + 'will allow calls to the Heat API resource-signal using the ' + 'provided keystone credentials. NO_SIGNAL will result in the ' + 'resource going to the COMPLETE state without waiting for ' + 'any signal.'), default=CFN_SIGNAL, constraints=[ constraints.AllowedValues(SIGNAL_TRANSPORTS), @@ -181,6 +185,10 @@ class SoftwareDeployment(signal_responder.SignalResponder): return self.properties.get( self.SIGNAL_TRANSPORT) == self.NO_SIGNAL + def _signal_transport_temp_url(self): + return self.properties.get( + self.SIGNAL_TRANSPORT) == self.TEMP_URL_SIGNAL + def _build_properties(self, properties, config_id, action): props = { 'config_id': config_id, @@ -283,6 +291,41 @@ class SoftwareDeployment(signal_responder.SignalResponder): def _build_derived_options(self, action, source): return source.get(sc.SoftwareConfig.OPTIONS) + def _get_temp_url(self): + put_url = self.data().get('signal_temp_url') + if put_url: + return put_url + + container = self.physical_resource_name() + object_name = str(uuid.uuid4()) + + self.client('swift').put_container(container) + + put_url = self.client_plugin('swift').get_temp_url( + container, object_name) + self.data_set('signal_temp_url', put_url) + self.data_set('signal_object_name', object_name) + + self.client('swift').put_object( + container, object_name, '') + return put_url + + def _delete_temp_url(self): + object_name = self.data().get('signal_object_name') + if not object_name: + return + try: + container = self.physical_resource_name() + swift = self.client('swift') + swift.delete_object(container, object_name) + headers = swift.head_container(container) + if int(headers['x-container-object-count']) == 0: + swift.delete_container(container) + except Exception as ex: + self.client_plugin('swift').ignore_not_found(ex) + self.data_delete('signal_object_name') + self.data_delete('signal_temp_url') + def _build_derived_inputs(self, action, source): scl = sc.SoftwareConfig inputs = copy.deepcopy(source.get(scl.INPUTS)) or [] @@ -323,15 +366,43 @@ class SoftwareDeployment(signal_responder.SignalResponder): 'stack'), scl.TYPE: 'String', 'value': self.name + }, { + scl.NAME: self.DEPLOY_SIGNAL_TRANSPORT, + scl.DESCRIPTION: _('How the server should signal to heat with ' + 'the deployment output values.'), + scl.TYPE: 'String', + 'value': self.properties.get(self.SIGNAL_TRANSPORT) }]) if self._signal_transport_cfn(): inputs.append({ scl.NAME: self.DEPLOY_SIGNAL_ID, - scl.DESCRIPTION: _('ID of signal to use for signalling ' + scl.DESCRIPTION: _('ID of signal to use for signaling ' 'output values'), scl.TYPE: 'String', 'value': self._get_signed_url() }) + inputs.append({ + scl.NAME: self.DEPLOY_SIGNAL_VERB, + scl.DESCRIPTION: _('HTTP verb to use for signaling ' + 'output values'), + scl.TYPE: 'String', + 'value': 'POST' + }) + elif self._signal_transport_temp_url(): + inputs.append({ + scl.NAME: self.DEPLOY_SIGNAL_ID, + scl.DESCRIPTION: _('ID of signal to use for signaling ' + 'output values'), + scl.TYPE: 'String', + 'value': self._get_temp_url() + }) + inputs.append({ + scl.NAME: self.DEPLOY_SIGNAL_VERB, + scl.DESCRIPTION: _('HTTP verb to use for signaling ' + 'output values'), + scl.TYPE: 'String', + 'value': 'PUT' + }) elif self._signal_transport_heat(): inputs.extend([{ scl.NAME: self.DEPLOY_AUTH_URL, @@ -416,6 +487,8 @@ class SoftwareDeployment(signal_responder.SignalResponder): self._delete_user() elif self._signal_transport_heat(): self._delete_user() + elif self._signal_transport_temp_url(): + self._delete_temp_url() derived_config_id = None if self.resource_id is not None: diff --git a/heat/engine/service.py b/heat/engine/service.py index 4bd7bd2f9..be43d73ed 100644 --- a/heat/engine/service.py +++ b/heat/engine/service.py @@ -27,6 +27,7 @@ from oslo_utils import uuidutils from osprofiler import profiler import requests import six +from six.moves.urllib import parse as urlparse import webob from heat.common import context @@ -1465,9 +1466,61 @@ class EngineService(service.Service): json_md = jsonutils.dumps(md) requests.put(metadata_put_url, json_md) + def _refresh_software_deployment(self, cnxt, sd, deploy_signal_id): + container, object_name = urlparse.urlparse( + deploy_signal_id).path.split('/')[-2:] + swift_plugin = cnxt.clients.client_plugin('swift') + swift = swift_plugin.client() + + try: + headers = swift.head_object(container, object_name) + except Exception as ex: + # ignore not-found, in case swift is not consistent yet + if swift_plugin.is_not_found(ex): + LOG.info(_LI('Signal object not found: %(c)s %(o)s') % { + 'c': container, 'o': object_name}) + return sd + raise ex + + lm = headers.get('last-modified') + + last_modified = swift_plugin.parse_last_modified(lm) + prev_last_modified = sd.updated_at + + if prev_last_modified: + # assume stored as utc, convert to offset-naive datetime + prev_last_modified = prev_last_modified.replace(tzinfo=None) + + if prev_last_modified and (last_modified <= prev_last_modified): + return sd + + try: + (headers, obj) = swift.get_object(container, object_name) + except Exception as ex: + # ignore not-found, in case swift is not consistent yet + if swift_plugin.is_not_found(ex): + LOG.info(_LI( + 'Signal object not found: %(c)s %(o)s') % { + 'c': container, 'o': object_name}) + return sd + raise ex + if obj: + self.signal_software_deployment( + cnxt, sd.id, json.loads(obj), + timeutils.strtime(last_modified)) + + return db_api.software_deployment_get(cnxt, sd.id) + @context.request_context def show_software_deployment(self, cnxt, deployment_id): sd = db_api.software_deployment_get(cnxt, deployment_id) + if sd.status == rpc_api.SOFTWARE_DEPLOYMENT_IN_PROGRESS: + c = sd.config.config + input_values = dict((i['name'], i['value']) for i in c['inputs']) + transport = input_values.get('deploy_signal_transport') + if transport == 'TEMP_URL_SIGNAL': + sd = self._refresh_software_deployment( + cnxt, sd, input_values.get('deploy_signal_id')) return api.format_software_deployment(sd) @context.request_context diff --git a/heat/tests/test_engine_service.py b/heat/tests/test_engine_service.py index ee2aef480..a69fa1e80 100644 --- a/heat/tests/test_engine_service.py +++ b/heat/tests/test_engine_service.py @@ -11,6 +11,7 @@ # License for the specific language governing permissions and limitations # under the License. +import datetime import functools import json import sys @@ -23,6 +24,7 @@ import mox from oslo_config import cfg from oslo_messaging.rpc import dispatcher from oslo_serialization import jsonutils +from oslo_utils import timeutils import six from heat.common import context @@ -34,6 +36,7 @@ from heat.db import api as db_api from heat.engine.clients.os import glance from heat.engine.clients.os import keystone from heat.engine.clients.os import nova +from heat.engine.clients.os import swift from heat.engine import dependencies from heat.engine import environment from heat.engine import properties @@ -3778,6 +3781,36 @@ class SoftwareConfigServiceTest(common.HeatTestCase): self.assertEqual(deployment_id, deployment['id']) self.assertEqual(kwargs['input_values'], deployment['input_values']) + @mock.patch.object(service.EngineService, '_refresh_software_deployment') + def test_show_software_deployment_refresh( + self, _refresh_software_deployment): + temp_url = ('http://192.0.2.1/v1/AUTH_a/b/c' + '?temp_url_sig=ctemp_url_expires=1234') + config = self._create_software_config(inputs=[ + { + 'name': 'deploy_signal_transport', + 'type': 'String', + 'value': 'TEMP_URL_SIGNAL' + }, { + 'name': 'deploy_signal_id', + 'type': 'String', + 'value': temp_url + } + ]) + + deployment = self._create_software_deployment( + status='IN_PROGRESS', config_id=config['id']) + + deployment_id = deployment['id'] + sd = db_api.software_deployment_get(self.ctx, deployment_id) + _refresh_software_deployment.return_value = sd + self.assertEqual( + deployment, + self.engine.show_software_deployment(self.ctx, deployment_id)) + self.assertEqual( + (self.ctx, sd, temp_url), + _refresh_software_deployment.call_args[0]) + def test_update_software_deployment_new_config(self): server_id = str(uuid.uuid4()) @@ -3934,6 +3967,111 @@ class SoftwareConfigServiceTest(common.HeatTestCase): put.assert_called_once_with( 'http://192.168.2.2/foo/bar', jsonutils.dumps(result_metadata)) + @mock.patch.object(service.EngineService, + 'signal_software_deployment') + @mock.patch.object(swift.SwiftClientPlugin, '_create') + def test_refresh_software_deployment(self, scc, ssd): + temp_url = ('http://192.0.2.1/v1/AUTH_a/b/c' + '?temp_url_sig=ctemp_url_expires=1234') + container = 'b' + object_name = 'c' + + config = self._create_software_config(inputs=[ + { + 'name': 'deploy_signal_transport', + 'type': 'String', + 'value': 'TEMP_URL_SIGNAL' + }, { + 'name': 'deploy_signal_id', + 'type': 'String', + 'value': temp_url + } + ]) + + timeutils.set_time_override( + datetime.datetime(2013, 1, 23, 22, 48, 5, 0)) + self.addCleanup(timeutils.clear_time_override) + now = timeutils.utcnow() + then = now - datetime.timedelta(0, 60) + + last_modified_1 = 'Wed, 23 Jan 2013 22:47:05 GMT' + last_modified_2 = 'Wed, 23 Jan 2013 22:48:05 GMT' + + sc = mock.MagicMock() + headers = { + 'last-modified': last_modified_1 + } + sc.head_object.return_value = headers + sc.get_object.return_value = (headers, '{"foo": "bar"}') + scc.return_value = sc + + deployment = self._create_software_deployment( + status='IN_PROGRESS', config_id=config['id']) + + deployment_id = six.text_type(deployment['id']) + sd = db_api.software_deployment_get(self.ctx, deployment_id) + + # poll with missing object + swift_exc = swift.SwiftClientPlugin.exceptions_module + sc.head_object.side_effect = swift_exc.ClientException( + 'Not found', http_status=404) + + self.assertEqual( + sd, + self.engine._refresh_software_deployment(self.ctx, sd, temp_url)) + sc.head_object.assert_called_once_with(container, object_name) + # no call to get_object or signal_last_modified + self.assertEqual([], sc.get_object.mock_calls) + self.assertEqual([], ssd.mock_calls) + + # poll with other error + sc.head_object.side_effect = swift_exc.ClientException( + 'Ouch', http_status=409) + self.assertRaises(swift_exc.ClientException, + self.engine._refresh_software_deployment, + self.ctx, sd, temp_url) + # no call to get_object or signal_last_modified + self.assertEqual([], sc.get_object.mock_calls) + self.assertEqual([], ssd.mock_calls) + sc.head_object.side_effect = None + + # first poll populates data signal_last_modified + self.engine._refresh_software_deployment(self.ctx, sd, temp_url) + sc.head_object.assert_called_with(container, object_name) + sc.get_object.assert_called_once_with(container, object_name) + # signal_software_deployment called with signal + ssd.assert_called_once_with(self.ctx, deployment_id, {u"foo": u"bar"}, + timeutils.strtime(then)) + + # second poll updated_at populated with first poll last-modified + db_api.software_deployment_update( + self.ctx, deployment_id, {'updated_at': then}) + sd = db_api.software_deployment_get(self.ctx, deployment_id) + self.assertEqual(then, sd.updated_at) + self.engine._refresh_software_deployment(self.ctx, sd, temp_url) + sc.get_object.assert_called_once_with(container, object_name) + # signal_software_deployment has not been called again + ssd.assert_called_once_with(self.ctx, deployment_id, {"foo": "bar"}, + timeutils.strtime(then)) + + # third poll last-modified changed, new signal + headers['last-modified'] = last_modified_2 + sc.head_object.return_value = headers + sc.get_object.return_value = (headers, '{"bar": "baz"}') + self.engine._refresh_software_deployment(self.ctx, sd, temp_url) + + # two calls to signal_software_deployment, for then and now + self.assertEqual(2, len(ssd.mock_calls)) + ssd.assert_called_with(self.ctx, deployment_id, {"bar": "baz"}, + timeutils.strtime(now)) + + # four polls result in only two signals, for then and now + db_api.software_deployment_update( + self.ctx, deployment_id, {'updated_at': now}) + sd = db_api.software_deployment_get(self.ctx, deployment_id) + self.engine._refresh_software_deployment(self.ctx, sd, temp_url) + self.assertEqual(2, len(ssd.mock_calls)) + class ThreadGroupManagerTest(common.HeatTestCase): def setUp(self): diff --git a/heat/tests/test_software_deployment.py b/heat/tests/test_software_deployment.py index c13726114..e69581e08 100644 --- a/heat/tests/test_software_deployment.py +++ b/heat/tests/test_software_deployment.py @@ -12,6 +12,8 @@ # under the License. import copy +import re +import uuid import mock import six @@ -19,6 +21,7 @@ import six from heat.common import exception as exc from heat.common.i18n import _ from heat.engine.clients.os import nova +from heat.engine.clients.os import swift from heat.engine import parser from heat.engine.resources.software_config import software_deployment as sd from heat.engine import rsrc_defn @@ -81,6 +84,22 @@ class SoftwareDeploymentTest(common.HeatTestCase): } } + template_temp_url_signal = { + 'HeatTemplateFormatVersion': '2012-12-12', + 'Resources': { + 'deployment_mysql': { + 'Type': 'OS::Heat::SoftwareDeployment', + 'Properties': { + 'server': '9f1f0e00-05d2-4ca5-8602-95021f19c9d0', + 'config': '48e8ade1-9196-42d5-89a2-f709fde42632', + 'input_values': {'foo': 'bar', 'bink': 'bonk'}, + 'signal_transport': 'TEMP_URL_SIGNAL', + 'name': '00_run_me_first' + } + } + } + } + template_delete_suspend_resume = { 'HeatTemplateFormatVersion': '2012-12-12', 'Resources': { @@ -281,6 +300,12 @@ class SoftwareDeploymentTest(common.HeatTestCase): 'name': 'deploy_resource_name', 'type': 'String', 'value': 'deployment_mysql' + }, { + 'description': ('How the server should signal to heat with ' + 'the deployment output values.'), + 'name': 'deploy_signal_transport', + 'type': 'String', + 'value': 'NO_SIGNAL' }], 'options': {}, 'outputs': [] @@ -372,6 +397,12 @@ class SoftwareDeploymentTest(common.HeatTestCase): 'name': 'deploy_resource_name', 'type': 'String', 'value': 'deployment_mysql' + }, { + 'description': ('How the server should signal to heat with ' + 'the deployment output values.'), + 'name': 'deploy_signal_transport', + 'type': 'String', + 'value': 'NO_SIGNAL' }], 'options': {}, 'outputs': [] @@ -814,6 +845,107 @@ class SoftwareDeploymentTest(common.HeatTestCase): self.assertIsNotNone(self.deployment.handle_resume()) self.assertIsNotNone(self.deployment.handle_delete()) + def test_get_temp_url(self): + dep_data = {} + + sc = mock.MagicMock() + scc = self.patch( + 'heat.engine.clients.os.swift.SwiftClientPlugin._create') + scc.return_value = sc + sc.head_account.return_value = { + 'x-account-meta-temp-url-key': 'secrit' + } + sc.url = 'http://192.0.2.1/v1/AUTH_test_tenant_id' + + self._create_stack(self.template_temp_url_signal) + + def data_set(key, value, redact=False): + dep_data[key] = value + + self.deployment.data_set = data_set + self.deployment.data = mock.Mock( + return_value=dep_data) + + self.deployment.id = str(uuid.uuid4()) + container = self.deployment.physical_resource_name() + + temp_url = self.deployment._get_temp_url() + temp_url_pattern = re.compile( + '^http://192.0.2.1/v1/AUTH_test_tenant_id/' + '(software_deployment_test_stack-deployment_mysql-.*)/(.*)' + '\\?temp_url_sig=.*&temp_url_expires=\\d*$') + self.assertRegex(temp_url, temp_url_pattern) + m = temp_url_pattern.search(temp_url) + object_name = m.group(2) + self.assertEqual(container, m.group(1)) + self.assertEqual(dep_data['signal_object_name'], object_name) + + self.assertEqual(dep_data['signal_temp_url'], temp_url) + + self.assertEqual(temp_url, self.deployment._get_temp_url()) + + sc.put_container.assert_called_once_with(container) + sc.put_object.assert_called_once_with(container, object_name, '') + + def test_delete_temp_url(self): + object_name = str(uuid.uuid4()) + dep_data = { + 'signal_object_name': object_name + } + self._create_stack(self.template_temp_url_signal) + + self.deployment.data_delete = mock.MagicMock() + self.deployment.data = mock.Mock( + return_value=dep_data) + + sc = mock.MagicMock() + sc.head_container.return_value = { + 'x-container-object-count': 0 + } + scc = self.patch( + 'heat.engine.clients.os.swift.SwiftClientPlugin._create') + scc.return_value = sc + + self.deployment.id = str(uuid.uuid4()) + container = self.deployment.physical_resource_name() + self.deployment._delete_temp_url() + sc.delete_object.assert_called_once_with(container, object_name) + self.assertEqual( + [mock.call('signal_object_name'), mock.call('signal_temp_url')], + self.deployment.data_delete.mock_calls) + + swift_exc = swift.SwiftClientPlugin.exceptions_module + sc.delete_object.side_effect = swift_exc.ClientException( + 'Not found', http_status=404) + self.deployment._delete_temp_url() + self.assertEqual( + [mock.call('signal_object_name'), mock.call('signal_temp_url'), + mock.call('signal_object_name'), mock.call('signal_temp_url')], + self.deployment.data_delete.mock_calls) + + del(dep_data['signal_object_name']) + self.deployment.physical_resource_name = mock.Mock() + self.deployment._delete_temp_url() + self.assertFalse(self.deployment.physical_resource_name.called) + + def test_handle_action_temp_url(self): + + self._create_stack(self.template_temp_url_signal) + dep_data = { + 'signal_temp_url': ( + 'http://192.0.2.1/v1/AUTH_a/b/c' + '?temp_url_sig=ctemp_url_expires=1234') + } + self.deployment.data = mock.Mock( + return_value=dep_data) + + self.mock_software_config() + + for action in ('DELETE', 'SUSPEND', 'RESUME'): + self.assertIsNone(self.deployment._handle_action(action)) + for action in ('CREATE', 'UPDATE'): + self.assertIsNotNone(self.deployment._handle_action(action)) + class SoftwareDeploymentsTest(common.HeatTestCase): diff --git a/heat/tests/test_swift_client.py b/heat/tests/test_swift_client.py index 4ea72a105..aeb95596b 100644 --- a/heat/tests/test_swift_client.py +++ b/heat/tests/test_swift_client.py @@ -11,7 +11,11 @@ # License for the specific language governing permissions and limitations # under the License. +import datetime + import mock +from oslo_utils import timeutils +import pytz from testtools import matchers from heat.engine.clients.os import swift @@ -120,3 +124,15 @@ class SwiftUtilsTests(SwiftClientPluginTestCase): "temp_url_expires=[0-9]{10}" % (container_name, obj_name)) self.assertThat(url, matchers.MatchesRegex(regexp)) + + def test_parse_last_modified(self): + self.assertIsNone(self.swift_plugin.parse_last_modified(None)) + now = datetime.datetime( + 2015, 2, 5, 1, 4, 40, 0, pytz.timezone('GMT')) + now_naive = datetime.datetime( + 2015, 2, 5, 1, 4, 40, 0) + last_modified = timeutils.strtime(now, '%a, %d %b %Y %H:%M:%S %Z') + self.assertEqual('Thu, 05 Feb 2015 01:04:40 GMT', last_modified) + self.assertEqual( + now_naive, + self.swift_plugin.parse_last_modified(last_modified))