Deployment signal_transport: TEMP_URL_SIGNAL

This change allows swift TempURLs to be used to signal heat from
server deployments.

When signal_transport: TEMP_URL_SIGNAL is specified, the following will happen:
* On creation a swift object will be created with an associated TempURL for PUTs
* The PUT URL is used for the deploy_signal_id input value, along with the new
  deploy_signal_verb input being set to PUT
* In the deployment resource check_complete, the swift object is polled for
  new signal contents, and signal() is called when the swift object contents change

By specifying deployment signal_transport: TEMP_URL_SIGNAL and
server software_config_transport: POLL_TEMP_URL it should now be possible to use
config/deployment resources with only a keystone v2 API (or with a standalone heat)

Implements-Blueprint: software-config-swift-signal

Change-Id: I1dfba248dcfc90c3d872ba35f0aa935cca5c5606
This commit is contained in:
Steve Baker 2015-02-12 11:33:19 +13:00
parent ea349ea48b
commit 7470a5e919
6 changed files with 439 additions and 9 deletions

View File

@ -11,6 +11,8 @@
# License for the specific language governing permissions and limitations
# under the License.
import datetime
import email.utils
import hashlib
import random
import time
@ -117,3 +119,19 @@ class SwiftClientPlugin(client_plugin.ClientPlugin):
self.client().put_object(container_name, obj_name, IN_PROGRESS)
return self.get_temp_url(container_name, obj_name, timeout)
def parse_last_modified(self, lm):
'''
Parses the last-modified value, such as from a swift object header,
and returns the datetime.datetime of that value.
:param lm: The last-modified value (or None)
:type lm: string
:returns: An offset-naive UTC datetime of the value (or None)
'''
if not lm:
return None
pd = email.utils.parsedate(lm)[:6]
# according to RFC 2616, all HTTP time headers must be
# in GMT time, so create an offset-naive UTC datetime
return datetime.datetime(*pd)

View File

@ -91,19 +91,21 @@ class SoftwareDeployment(signal_responder.SignalResponder):
DEPLOY_SIGNAL_ID, DEPLOY_STACK_ID,
DEPLOY_RESOURCE_NAME, DEPLOY_AUTH_URL,
DEPLOY_USERNAME, DEPLOY_PASSWORD,
DEPLOY_PROJECT_ID, DEPLOY_USER_ID
DEPLOY_PROJECT_ID, DEPLOY_USER_ID,
DEPLOY_SIGNAL_VERB, DEPLOY_SIGNAL_TRANSPORT
) = (
'deploy_server_id', 'deploy_action',
'deploy_signal_id', 'deploy_stack_id',
'deploy_resource_name', 'deploy_auth_url',
'deploy_username', 'deploy_password',
'deploy_project_id', 'deploy_user_id'
'deploy_project_id', 'deploy_user_id',
'deploy_signal_verb', 'deploy_signal_transport'
)
SIGNAL_TRANSPORTS = (
CFN_SIGNAL, HEAT_SIGNAL, NO_SIGNAL
CFN_SIGNAL, TEMP_URL_SIGNAL, HEAT_SIGNAL, NO_SIGNAL
) = (
'CFN_SIGNAL', 'HEAT_SIGNAL', 'NO_SIGNAL'
'CFN_SIGNAL', 'TEMP_URL_SIGNAL', 'HEAT_SIGNAL', 'NO_SIGNAL'
)
properties_schema = {
@ -144,10 +146,12 @@ class SoftwareDeployment(signal_responder.SignalResponder):
properties.Schema.STRING,
_('How the server should signal to heat with the deployment '
'output values. CFN_SIGNAL will allow an HTTP POST to a CFN '
'keypair signed URL. HEAT_SIGNAL will allow calls to '
'the Heat API resource-signal using the provided keystone '
'credentials. NO_SIGNAL will result in the resource going to '
'the COMPLETE state without waiting for any signal.'),
'keypair signed URL. TEMP_URL_SIGNAL will create a '
'Swift TempURL to be signaled via HTTP PUT. HEAT_SIGNAL '
'will allow calls to the Heat API resource-signal using the '
'provided keystone credentials. NO_SIGNAL will result in the '
'resource going to the COMPLETE state without waiting for '
'any signal.'),
default=CFN_SIGNAL,
constraints=[
constraints.AllowedValues(SIGNAL_TRANSPORTS),
@ -181,6 +185,10 @@ class SoftwareDeployment(signal_responder.SignalResponder):
return self.properties.get(
self.SIGNAL_TRANSPORT) == self.NO_SIGNAL
def _signal_transport_temp_url(self):
return self.properties.get(
self.SIGNAL_TRANSPORT) == self.TEMP_URL_SIGNAL
def _build_properties(self, properties, config_id, action):
props = {
'config_id': config_id,
@ -283,6 +291,41 @@ class SoftwareDeployment(signal_responder.SignalResponder):
def _build_derived_options(self, action, source):
return source.get(sc.SoftwareConfig.OPTIONS)
def _get_temp_url(self):
put_url = self.data().get('signal_temp_url')
if put_url:
return put_url
container = self.physical_resource_name()
object_name = str(uuid.uuid4())
self.client('swift').put_container(container)
put_url = self.client_plugin('swift').get_temp_url(
container, object_name)
self.data_set('signal_temp_url', put_url)
self.data_set('signal_object_name', object_name)
self.client('swift').put_object(
container, object_name, '')
return put_url
def _delete_temp_url(self):
object_name = self.data().get('signal_object_name')
if not object_name:
return
try:
container = self.physical_resource_name()
swift = self.client('swift')
swift.delete_object(container, object_name)
headers = swift.head_container(container)
if int(headers['x-container-object-count']) == 0:
swift.delete_container(container)
except Exception as ex:
self.client_plugin('swift').ignore_not_found(ex)
self.data_delete('signal_object_name')
self.data_delete('signal_temp_url')
def _build_derived_inputs(self, action, source):
scl = sc.SoftwareConfig
inputs = copy.deepcopy(source.get(scl.INPUTS)) or []
@ -323,15 +366,43 @@ class SoftwareDeployment(signal_responder.SignalResponder):
'stack'),
scl.TYPE: 'String',
'value': self.name
}, {
scl.NAME: self.DEPLOY_SIGNAL_TRANSPORT,
scl.DESCRIPTION: _('How the server should signal to heat with '
'the deployment output values.'),
scl.TYPE: 'String',
'value': self.properties.get(self.SIGNAL_TRANSPORT)
}])
if self._signal_transport_cfn():
inputs.append({
scl.NAME: self.DEPLOY_SIGNAL_ID,
scl.DESCRIPTION: _('ID of signal to use for signalling '
scl.DESCRIPTION: _('ID of signal to use for signaling '
'output values'),
scl.TYPE: 'String',
'value': self._get_signed_url()
})
inputs.append({
scl.NAME: self.DEPLOY_SIGNAL_VERB,
scl.DESCRIPTION: _('HTTP verb to use for signaling '
'output values'),
scl.TYPE: 'String',
'value': 'POST'
})
elif self._signal_transport_temp_url():
inputs.append({
scl.NAME: self.DEPLOY_SIGNAL_ID,
scl.DESCRIPTION: _('ID of signal to use for signaling '
'output values'),
scl.TYPE: 'String',
'value': self._get_temp_url()
})
inputs.append({
scl.NAME: self.DEPLOY_SIGNAL_VERB,
scl.DESCRIPTION: _('HTTP verb to use for signaling '
'output values'),
scl.TYPE: 'String',
'value': 'PUT'
})
elif self._signal_transport_heat():
inputs.extend([{
scl.NAME: self.DEPLOY_AUTH_URL,
@ -416,6 +487,8 @@ class SoftwareDeployment(signal_responder.SignalResponder):
self._delete_user()
elif self._signal_transport_heat():
self._delete_user()
elif self._signal_transport_temp_url():
self._delete_temp_url()
derived_config_id = None
if self.resource_id is not None:

View File

@ -27,6 +27,7 @@ from oslo_utils import uuidutils
from osprofiler import profiler
import requests
import six
from six.moves.urllib import parse as urlparse
import webob
from heat.common import context
@ -1465,9 +1466,61 @@ class EngineService(service.Service):
json_md = jsonutils.dumps(md)
requests.put(metadata_put_url, json_md)
def _refresh_software_deployment(self, cnxt, sd, deploy_signal_id):
container, object_name = urlparse.urlparse(
deploy_signal_id).path.split('/')[-2:]
swift_plugin = cnxt.clients.client_plugin('swift')
swift = swift_plugin.client()
try:
headers = swift.head_object(container, object_name)
except Exception as ex:
# ignore not-found, in case swift is not consistent yet
if swift_plugin.is_not_found(ex):
LOG.info(_LI('Signal object not found: %(c)s %(o)s') % {
'c': container, 'o': object_name})
return sd
raise ex
lm = headers.get('last-modified')
last_modified = swift_plugin.parse_last_modified(lm)
prev_last_modified = sd.updated_at
if prev_last_modified:
# assume stored as utc, convert to offset-naive datetime
prev_last_modified = prev_last_modified.replace(tzinfo=None)
if prev_last_modified and (last_modified <= prev_last_modified):
return sd
try:
(headers, obj) = swift.get_object(container, object_name)
except Exception as ex:
# ignore not-found, in case swift is not consistent yet
if swift_plugin.is_not_found(ex):
LOG.info(_LI(
'Signal object not found: %(c)s %(o)s') % {
'c': container, 'o': object_name})
return sd
raise ex
if obj:
self.signal_software_deployment(
cnxt, sd.id, json.loads(obj),
timeutils.strtime(last_modified))
return db_api.software_deployment_get(cnxt, sd.id)
@context.request_context
def show_software_deployment(self, cnxt, deployment_id):
sd = db_api.software_deployment_get(cnxt, deployment_id)
if sd.status == rpc_api.SOFTWARE_DEPLOYMENT_IN_PROGRESS:
c = sd.config.config
input_values = dict((i['name'], i['value']) for i in c['inputs'])
transport = input_values.get('deploy_signal_transport')
if transport == 'TEMP_URL_SIGNAL':
sd = self._refresh_software_deployment(
cnxt, sd, input_values.get('deploy_signal_id'))
return api.format_software_deployment(sd)
@context.request_context

View File

@ -11,6 +11,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import datetime
import functools
import json
import sys
@ -23,6 +24,7 @@ import mox
from oslo_config import cfg
from oslo_messaging.rpc import dispatcher
from oslo_serialization import jsonutils
from oslo_utils import timeutils
import six
from heat.common import context
@ -34,6 +36,7 @@ from heat.db import api as db_api
from heat.engine.clients.os import glance
from heat.engine.clients.os import keystone
from heat.engine.clients.os import nova
from heat.engine.clients.os import swift
from heat.engine import dependencies
from heat.engine import environment
from heat.engine import properties
@ -3778,6 +3781,36 @@ class SoftwareConfigServiceTest(common.HeatTestCase):
self.assertEqual(deployment_id, deployment['id'])
self.assertEqual(kwargs['input_values'], deployment['input_values'])
@mock.patch.object(service.EngineService, '_refresh_software_deployment')
def test_show_software_deployment_refresh(
self, _refresh_software_deployment):
temp_url = ('http://192.0.2.1/v1/AUTH_a/b/c'
'?temp_url_sig=ctemp_url_expires=1234')
config = self._create_software_config(inputs=[
{
'name': 'deploy_signal_transport',
'type': 'String',
'value': 'TEMP_URL_SIGNAL'
}, {
'name': 'deploy_signal_id',
'type': 'String',
'value': temp_url
}
])
deployment = self._create_software_deployment(
status='IN_PROGRESS', config_id=config['id'])
deployment_id = deployment['id']
sd = db_api.software_deployment_get(self.ctx, deployment_id)
_refresh_software_deployment.return_value = sd
self.assertEqual(
deployment,
self.engine.show_software_deployment(self.ctx, deployment_id))
self.assertEqual(
(self.ctx, sd, temp_url),
_refresh_software_deployment.call_args[0])
def test_update_software_deployment_new_config(self):
server_id = str(uuid.uuid4())
@ -3934,6 +3967,111 @@ class SoftwareConfigServiceTest(common.HeatTestCase):
put.assert_called_once_with(
'http://192.168.2.2/foo/bar', jsonutils.dumps(result_metadata))
@mock.patch.object(service.EngineService,
'signal_software_deployment')
@mock.patch.object(swift.SwiftClientPlugin, '_create')
def test_refresh_software_deployment(self, scc, ssd):
temp_url = ('http://192.0.2.1/v1/AUTH_a/b/c'
'?temp_url_sig=ctemp_url_expires=1234')
container = 'b'
object_name = 'c'
config = self._create_software_config(inputs=[
{
'name': 'deploy_signal_transport',
'type': 'String',
'value': 'TEMP_URL_SIGNAL'
}, {
'name': 'deploy_signal_id',
'type': 'String',
'value': temp_url
}
])
timeutils.set_time_override(
datetime.datetime(2013, 1, 23, 22, 48, 5, 0))
self.addCleanup(timeutils.clear_time_override)
now = timeutils.utcnow()
then = now - datetime.timedelta(0, 60)
last_modified_1 = 'Wed, 23 Jan 2013 22:47:05 GMT'
last_modified_2 = 'Wed, 23 Jan 2013 22:48:05 GMT'
sc = mock.MagicMock()
headers = {
'last-modified': last_modified_1
}
sc.head_object.return_value = headers
sc.get_object.return_value = (headers, '{"foo": "bar"}')
scc.return_value = sc
deployment = self._create_software_deployment(
status='IN_PROGRESS', config_id=config['id'])
deployment_id = six.text_type(deployment['id'])
sd = db_api.software_deployment_get(self.ctx, deployment_id)
# poll with missing object
swift_exc = swift.SwiftClientPlugin.exceptions_module
sc.head_object.side_effect = swift_exc.ClientException(
'Not found', http_status=404)
self.assertEqual(
sd,
self.engine._refresh_software_deployment(self.ctx, sd, temp_url))
sc.head_object.assert_called_once_with(container, object_name)
# no call to get_object or signal_last_modified
self.assertEqual([], sc.get_object.mock_calls)
self.assertEqual([], ssd.mock_calls)
# poll with other error
sc.head_object.side_effect = swift_exc.ClientException(
'Ouch', http_status=409)
self.assertRaises(swift_exc.ClientException,
self.engine._refresh_software_deployment,
self.ctx, sd, temp_url)
# no call to get_object or signal_last_modified
self.assertEqual([], sc.get_object.mock_calls)
self.assertEqual([], ssd.mock_calls)
sc.head_object.side_effect = None
# first poll populates data signal_last_modified
self.engine._refresh_software_deployment(self.ctx, sd, temp_url)
sc.head_object.assert_called_with(container, object_name)
sc.get_object.assert_called_once_with(container, object_name)
# signal_software_deployment called with signal
ssd.assert_called_once_with(self.ctx, deployment_id, {u"foo": u"bar"},
timeutils.strtime(then))
# second poll updated_at populated with first poll last-modified
db_api.software_deployment_update(
self.ctx, deployment_id, {'updated_at': then})
sd = db_api.software_deployment_get(self.ctx, deployment_id)
self.assertEqual(then, sd.updated_at)
self.engine._refresh_software_deployment(self.ctx, sd, temp_url)
sc.get_object.assert_called_once_with(container, object_name)
# signal_software_deployment has not been called again
ssd.assert_called_once_with(self.ctx, deployment_id, {"foo": "bar"},
timeutils.strtime(then))
# third poll last-modified changed, new signal
headers['last-modified'] = last_modified_2
sc.head_object.return_value = headers
sc.get_object.return_value = (headers, '{"bar": "baz"}')
self.engine._refresh_software_deployment(self.ctx, sd, temp_url)
# two calls to signal_software_deployment, for then and now
self.assertEqual(2, len(ssd.mock_calls))
ssd.assert_called_with(self.ctx, deployment_id, {"bar": "baz"},
timeutils.strtime(now))
# four polls result in only two signals, for then and now
db_api.software_deployment_update(
self.ctx, deployment_id, {'updated_at': now})
sd = db_api.software_deployment_get(self.ctx, deployment_id)
self.engine._refresh_software_deployment(self.ctx, sd, temp_url)
self.assertEqual(2, len(ssd.mock_calls))
class ThreadGroupManagerTest(common.HeatTestCase):
def setUp(self):

View File

@ -12,6 +12,8 @@
# under the License.
import copy
import re
import uuid
import mock
import six
@ -19,6 +21,7 @@ import six
from heat.common import exception as exc
from heat.common.i18n import _
from heat.engine.clients.os import nova
from heat.engine.clients.os import swift
from heat.engine import parser
from heat.engine.resources.software_config import software_deployment as sd
from heat.engine import rsrc_defn
@ -81,6 +84,22 @@ class SoftwareDeploymentTest(common.HeatTestCase):
}
}
template_temp_url_signal = {
'HeatTemplateFormatVersion': '2012-12-12',
'Resources': {
'deployment_mysql': {
'Type': 'OS::Heat::SoftwareDeployment',
'Properties': {
'server': '9f1f0e00-05d2-4ca5-8602-95021f19c9d0',
'config': '48e8ade1-9196-42d5-89a2-f709fde42632',
'input_values': {'foo': 'bar', 'bink': 'bonk'},
'signal_transport': 'TEMP_URL_SIGNAL',
'name': '00_run_me_first'
}
}
}
}
template_delete_suspend_resume = {
'HeatTemplateFormatVersion': '2012-12-12',
'Resources': {
@ -281,6 +300,12 @@ class SoftwareDeploymentTest(common.HeatTestCase):
'name': 'deploy_resource_name',
'type': 'String',
'value': 'deployment_mysql'
}, {
'description': ('How the server should signal to heat with '
'the deployment output values.'),
'name': 'deploy_signal_transport',
'type': 'String',
'value': 'NO_SIGNAL'
}],
'options': {},
'outputs': []
@ -372,6 +397,12 @@ class SoftwareDeploymentTest(common.HeatTestCase):
'name': 'deploy_resource_name',
'type': 'String',
'value': 'deployment_mysql'
}, {
'description': ('How the server should signal to heat with '
'the deployment output values.'),
'name': 'deploy_signal_transport',
'type': 'String',
'value': 'NO_SIGNAL'
}],
'options': {},
'outputs': []
@ -814,6 +845,107 @@ class SoftwareDeploymentTest(common.HeatTestCase):
self.assertIsNotNone(self.deployment.handle_resume())
self.assertIsNotNone(self.deployment.handle_delete())
def test_get_temp_url(self):
dep_data = {}
sc = mock.MagicMock()
scc = self.patch(
'heat.engine.clients.os.swift.SwiftClientPlugin._create')
scc.return_value = sc
sc.head_account.return_value = {
'x-account-meta-temp-url-key': 'secrit'
}
sc.url = 'http://192.0.2.1/v1/AUTH_test_tenant_id'
self._create_stack(self.template_temp_url_signal)
def data_set(key, value, redact=False):
dep_data[key] = value
self.deployment.data_set = data_set
self.deployment.data = mock.Mock(
return_value=dep_data)
self.deployment.id = str(uuid.uuid4())
container = self.deployment.physical_resource_name()
temp_url = self.deployment._get_temp_url()
temp_url_pattern = re.compile(
'^http://192.0.2.1/v1/AUTH_test_tenant_id/'
'(software_deployment_test_stack-deployment_mysql-.*)/(.*)'
'\\?temp_url_sig=.*&temp_url_expires=\\d*$')
self.assertRegex(temp_url, temp_url_pattern)
m = temp_url_pattern.search(temp_url)
object_name = m.group(2)
self.assertEqual(container, m.group(1))
self.assertEqual(dep_data['signal_object_name'], object_name)
self.assertEqual(dep_data['signal_temp_url'], temp_url)
self.assertEqual(temp_url, self.deployment._get_temp_url())
sc.put_container.assert_called_once_with(container)
sc.put_object.assert_called_once_with(container, object_name, '')
def test_delete_temp_url(self):
object_name = str(uuid.uuid4())
dep_data = {
'signal_object_name': object_name
}
self._create_stack(self.template_temp_url_signal)
self.deployment.data_delete = mock.MagicMock()
self.deployment.data = mock.Mock(
return_value=dep_data)
sc = mock.MagicMock()
sc.head_container.return_value = {
'x-container-object-count': 0
}
scc = self.patch(
'heat.engine.clients.os.swift.SwiftClientPlugin._create')
scc.return_value = sc
self.deployment.id = str(uuid.uuid4())
container = self.deployment.physical_resource_name()
self.deployment._delete_temp_url()
sc.delete_object.assert_called_once_with(container, object_name)
self.assertEqual(
[mock.call('signal_object_name'), mock.call('signal_temp_url')],
self.deployment.data_delete.mock_calls)
swift_exc = swift.SwiftClientPlugin.exceptions_module
sc.delete_object.side_effect = swift_exc.ClientException(
'Not found', http_status=404)
self.deployment._delete_temp_url()
self.assertEqual(
[mock.call('signal_object_name'), mock.call('signal_temp_url'),
mock.call('signal_object_name'), mock.call('signal_temp_url')],
self.deployment.data_delete.mock_calls)
del(dep_data['signal_object_name'])
self.deployment.physical_resource_name = mock.Mock()
self.deployment._delete_temp_url()
self.assertFalse(self.deployment.physical_resource_name.called)
def test_handle_action_temp_url(self):
self._create_stack(self.template_temp_url_signal)
dep_data = {
'signal_temp_url': (
'http://192.0.2.1/v1/AUTH_a/b/c'
'?temp_url_sig=ctemp_url_expires=1234')
}
self.deployment.data = mock.Mock(
return_value=dep_data)
self.mock_software_config()
for action in ('DELETE', 'SUSPEND', 'RESUME'):
self.assertIsNone(self.deployment._handle_action(action))
for action in ('CREATE', 'UPDATE'):
self.assertIsNotNone(self.deployment._handle_action(action))
class SoftwareDeploymentsTest(common.HeatTestCase):

View File

@ -11,7 +11,11 @@
# License for the specific language governing permissions and limitations
# under the License.
import datetime
import mock
from oslo_utils import timeutils
import pytz
from testtools import matchers
from heat.engine.clients.os import swift
@ -120,3 +124,15 @@ class SwiftUtilsTests(SwiftClientPluginTestCase):
"temp_url_expires=[0-9]{10}" %
(container_name, obj_name))
self.assertThat(url, matchers.MatchesRegex(regexp))
def test_parse_last_modified(self):
self.assertIsNone(self.swift_plugin.parse_last_modified(None))
now = datetime.datetime(
2015, 2, 5, 1, 4, 40, 0, pytz.timezone('GMT'))
now_naive = datetime.datetime(
2015, 2, 5, 1, 4, 40, 0)
last_modified = timeutils.strtime(now, '%a, %d %b %Y %H:%M:%S %Z')
self.assertEqual('Thu, 05 Feb 2015 01:04:40 GMT', last_modified)
self.assertEqual(
now_naive,
self.swift_plugin.parse_last_modified(last_modified))