openstack
/
heat
Code Issues Proposed changes

Add autoscaling notifications 

Send 2 notifications when an auto-scaling event occurs :
- successful resizing : 'autoscaling.start' and 'autoscaling.end'
- failed resizing : 'autoscaling.start' and 'autoscaling.error'

Implements blueprint send-notification-autoscaling

In same time, split out all notification code in its own module :
- stack CRUD : heat/engine/notification/stack.py
- autoscaling events : heat/engine/notification/autoscaling.py

Co-Authored-By: Angus Salkeld <angus@salkeld.id.au>
(from https://review.openstack.org/#/c/54661/)

Change-Id: I23e4478faf58ef23b031a3b0d4087c892dc4e96f
This commit is contained in:
Swann Croiset 2013-12-11 17:04:35 +01:00 committed by Swann Croiset
parent 0ba5b95cb1
commit 05592fa260
8 changed files with 359 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
heat/engine/notification/__init__.py Normal file
View File

@ -0,0 +1,43 @@
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo.config import cfg
from heat.openstack.common import log
from heat.openstack.common.notifier import api as notifier_api
LOG = log.getLogger(__name__)
SERVICE = 'orchestration'
CONF = cfg.CONF
CONF.import_opt('default_notification_level',
'heat.openstack.common.notifier.api')
CONF.import_opt('default_publisher_id',
'heat.openstack.common.notifier.api')
def _get_default_publisher():
publisher_id = CONF.default_publisher_id
if publisher_id is None:
publisher_id = notifier_api.publisher_id(SERVICE)
return publisher_id
def get_default_level():
return CONF.default_notification_level.upper()
def notify(context, event_type, level, body):
notifier_api.notify(context, _get_default_publisher(),
"%s.%s" % (SERVICE, event_type),
level, body)

43
heat/engine/notification/autoscaling.py Normal file
View File

@ -0,0 +1,43 @@
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from heat.openstack.common.notifier import api as notifier_api
from heat.engine import api as engine_api
from heat.engine import notification
def send(stack,
adjustment=None,
adjustment_type=None,
capacity=None,
groupname=None,
message='error',
suffix=None):
"""Send autoscaling notifications to the configured notification driver."""
# see: https://wiki.openstack.org/wiki/SystemUsageData
event_type = '%s.%s' % ('autoscaling', suffix)
body = engine_api.format_notification_body(stack)
body['adjustment_type'] = adjustment_type
body['adjustment'] = adjustment
body['capacity'] = capacity
body['groupname'] = groupname
body['message'] = message
level = notification.get_default_level()
if suffix == 'error':
level = notifier_api.ERROR
notification.notify(stack.context, event_type, level, body)

25
heat/engine/notification.py → heat/engine/notification/stack.py
View File

@ -11,33 +11,20 @@
# License for the specific language governing permissions and limitations
# under the License.
from oslo.config import cfg
from heat.openstack.common import log
from heat.openstack.common.notifier import api as notifier_api
from heat.engine import api as engine_api
LOG = log.getLogger(__name__)
SERVICE = 'orchestration'
CONF = cfg.CONF
CONF.import_opt('default_notification_level',
'heat.openstack.common.notifier.api')
CONF.import_opt('default_publisher_id',
'heat.openstack.common.notifier.api')
from heat.engine import api as engine_api
from heat.engine import notification
def send(stack):
"""Send usage notifications to the configured notification driver."""
publisher_id = CONF.default_publisher_id
if publisher_id is None:
publisher_id = notifier_api.publisher_id(SERVICE)
# The current notifications have a start/end:
# see: https://wiki.openstack.org/wiki/SystemUsageData
# so to be consistant we translate our status into a known start/end/error
# suffix.
level = CONF.default_notification_level.upper()
level = notification.get_default_level()
if stack.status == stack.IN_PROGRESS:
suffix = 'start'
elif stack.status == stack.COMPLETE:
@ -46,8 +33,8 @@ def send(stack):
suffix = 'error'
level = notifier_api.ERROR
event_type = '%s.%s.%s' % (SERVICE, stack.action.lower(), suffix)
event_type = '%s.%s' % (stack.action.lower(),
suffix)
notifier_api.notify(stack.context, publisher_id,
event_type, level,
notification.notify(stack.context, event_type, level,
engine_api.format_notification_body(stack))

2
heat/engine/parser.py
View File

@ -25,13 +25,13 @@ from heat.engine import environment
from heat.common import exception
from heat.engine import dependencies
from heat.common import identifier
from heat.engine import notification
from heat.engine import resource
from heat.engine import resources
from heat.engine import scheduler
from heat.engine import template
from heat.engine import timestamp
from heat.engine import update
from heat.engine.notification import stack as notification
from heat.engine.parameters import Parameters
from heat.engine.template import Template
from heat.engine.clients import Clients

36
heat/engine/resources/autoscaling.py
View File

@ -25,10 +25,12 @@ from heat.engine import signal_responder
from heat.common import short_id
from heat.common import exception
from heat.common import timeutils as iso8601utils
from heat.openstack.common import excutils
from heat.openstack.common import log as logging
from heat.openstack.common import timeutils
from heat.engine.properties import Properties
from heat.engine import constraints
from heat.engine.notification import autoscaling as notification
from heat.engine import properties
from heat.engine import scheduler
from heat.engine import stack_resource
@ -632,12 +634,40 @@ class AutoScalingGroup(InstanceGroup, CooldownMixin):
logger.debug(_('no change in capacity %d') % capacity)
return
result = self.resize(new_capacity)
# send a notification before, on-error and on-success.
notif = {
'stack': self.stack,
'adjustment': adjustment,
'adjustment_type': adjustment_type,
'capacity': capacity,
'groupname': self.FnGetRefId(),
'message': _("Start resizing the group %(group)s") % {
'group': self.FnGetRefId()},
'suffix': 'start',
}
notification.send(**notif)
try:
self.resize(new_capacity)
except Exception as resize_ex:
with excutils.save_and_reraise_exception():
try:
notif.update({'suffix': 'error',
'message': str(resize_ex),
})
notification.send(**notif)
except Exception:
logger.exception(_('Failed sending error notification'))
else:
notif.update({
'suffix': 'end',
'capacity': new_capacity,
'message': _("End resizing the group %(group)s") % {
'group': notif['groupname']},
})
notification.send(**notif)
self._cooldown_timestamp("%s : %s" % (adjustment_type, adjustment))
return result
def _tags(self):
"""Add Identifing Tags to all servers in the group.

55
heat/tests/test_autoscaling.py
View File

@ -28,6 +28,7 @@ from heat.engine.resources import autoscaling as asc
from heat.engine.resources import loadbalancer
from heat.engine.resources import instance
from heat.engine.resources.neutron import loadbalancer as neutron_lb
from heat.engine.notification import autoscaling as notification
from heat.engine import parser
from heat.engine import resource
from heat.engine import scheduler
@ -185,6 +186,45 @@ class AutoScalingTest(HeatTestCase):
mox.IgnoreArg(), mox.IgnoreArg(),
{'Instances': expected_list}).AndReturn(None)
def _stub_scale_notification(self,
adjust,
groupname,
start_capacity,
adjust_type='ChangeInCapacity',
end_capacity=None,
with_error=None):
self.m.StubOutWithMock(notification, 'send')
notification.send(stack=mox.IgnoreArg(),
adjustment=adjust,
adjustment_type=adjust_type,
capacity=start_capacity,
groupname=mox.IgnoreArg(),
suffix='start',
message="Start resizing the group %s"
% groupname,
).AndReturn(False)
if with_error:
notification.send(stack=mox.IgnoreArg(),
adjustment=adjust,
capacity=start_capacity,
adjustment_type=adjust_type,
groupname=mox.IgnoreArg(),
message='Nested stack update failed:'
' Error: %s' % with_error,
suffix='error',
).AndReturn(False)
else:
notification.send(stack=mox.IgnoreArg(),
adjustment=adjust,
adjustment_type=adjust_type,
capacity=end_capacity,
groupname=mox.IgnoreArg(),
message="End resizing the group %s"
% groupname,
suffix='end',
).AndReturn(False)
def _stub_meta_expected(self, now, data, nmeta=1):
# Stop time at now
self.m.StubOutWithMock(timeutils, 'utcnow')
@ -238,6 +278,8 @@ class AutoScalingTest(HeatTestCase):
# trigger adjustment to reduce to 0, there should be no more instances
self._stub_lb_reload(0)
self._stub_scale_notification(adjust=-1, groupname=rsrc.FnGetRefId(),
start_capacity=1, end_capacity=0)
self._stub_meta_expected(now, 'ChangeInCapacity : -1')
self.m.ReplayAll()
rsrc.adjust(-1)
@ -767,6 +809,8 @@ class AutoScalingTest(HeatTestCase):
self._stub_lb_reload(1)
self._stub_validate()
self._stub_meta_expected(now, 'ChangeInCapacity : -2')
self._stub_scale_notification(adjust=-2, groupname=rsrc.FnGetRefId(),
start_capacity=3, end_capacity=1)
self.m.ReplayAll()
rsrc.adjust(-2)
self.assertEqual(1, len(rsrc.get_instance_names()))
@ -775,6 +819,8 @@ class AutoScalingTest(HeatTestCase):
self._stub_lb_reload(3)
self._stub_meta_expected(now, 'ChangeInCapacity : 2')
self._stub_create(2)
self._stub_scale_notification(adjust=2, groupname=rsrc.FnGetRefId(),
start_capacity=1, end_capacity=3)
self.m.ReplayAll()
rsrc.adjust(2)
self.assertEqual(3, len(rsrc.get_instance_names()))
@ -783,6 +829,9 @@ class AutoScalingTest(HeatTestCase):
self._stub_lb_reload(2)
self._stub_validate()
self._stub_meta_expected(now, 'ExactCapacity : 2')
self._stub_scale_notification(adjust=2, groupname=rsrc.FnGetRefId(),
adjust_type='ExactCapacity',
start_capacity=3, end_capacity=2)
self.m.ReplayAll()
rsrc.adjust(2, 'ExactCapacity')
self.assertEqual(2, len(rsrc.get_instance_names()))
@ -805,9 +854,13 @@ class AutoScalingTest(HeatTestCase):
# Scale up one 1 instance with resource failure
self.m.StubOutWithMock(instance.Instance, 'handle_create')
instance.Instance.handle_create().AndRaise(exception.Error())
instance.Instance.handle_create().AndRaise(exception.Error('Bang'))
self._stub_lb_reload(1, unset=False, nochange=True)
self._stub_validate()
self._stub_scale_notification(adjust=1,
groupname=rsrc.FnGetRefId(),
start_capacity=1,
with_error='Bang')
self.m.ReplayAll()
self.assertRaises(exception.Error, rsrc.adjust, 1)

2
heat/tests/test_autoscaling_update_policy.py
View File

@ -20,7 +20,7 @@ from oslo.config import cfg
from heat.common import exception
from heat.common import template_format
from heat.engine import notification
from heat.engine.notification import stack as notification
from heat.engine import parser
from heat.engine.resources import user
from heat.engine.resources import instance

179
heat/tests/test_notifications.py
View File

@ -16,14 +16,28 @@ from oslo.config import cfg
from heat.openstack.common import timeutils
from heat.common import exception
from heat.common import template_format
from heat.engine import environment
from heat.engine import parser
from heat.engine import resource
# imports for mocking
from heat.engine import signal_responder as signal
from heat.engine import stack_resource
from heat.engine.resources import autoscaling
from heat.engine.resources import instance
from heat.engine.resources import loadbalancer
from heat.engine.resources import user
from heat.engine.resources import wait_condition as waitc
from heat.tests import generic_resource
from heat.tests import utils
from heat.tests import common
# reuse the same template than autoscaling tests
from heat.tests.test_autoscaling import as_template
class NotificationTest(common.HeatTestCase):
@ -140,5 +154,168 @@ class NotificationTest(common.HeatTestCase):
self.stack.state)
expected = self.expected['create'] + self.expected['delete']
expected = self.expected['create'] + self.expected['delete']
self.assertEqual(expected, mock_notify.call_args_list)
class ScaleNotificationTest(common.HeatTestCase):
def setUp(self):
super(ScaleNotificationTest, self).setUp()
utils.setup_dummy_db()
cfg.CONF.import_opt('notification_driver',
'heat.openstack.common.notifier.api')
cfg.CONF.set_default('notification_driver',
['heat.openstack.common.notifier.test_notifier'])
cfg.CONF.set_default('host', 'test_host')
self.ctx = utils.dummy_context()
self.ctx.tenant_id = 'test_tenant'
def create_autoscaling_stack_and_get_group(self):
env = environment.Environment()
env.load({u'parameters':
{u'KeyName': 'foo', 'ImageId': 'cloudimage'}})
t = template_format.parse(as_template)
template = parser.Template(t)
self.stack_name = utils.random_name()
stack = parser.Stack(self.ctx, self.stack_name, template,
env=env, disable_rollback=True)
stack.store()
self.created_time = stack.created_time
self.create_at = timeutils.isotime(self.created_time)
stack.create()
self.stack = stack
group = stack['WebServerGroup']
self.assertEqual((group.CREATE, group.COMPLETE), group.state)
return group
def mock_stack_except_for_group(self):
self.m_validate = self.patchobject(parser.Stack, 'validate')
self.patchobject(instance.Instance, 'handle_create')\
.return_value = True
self.patchobject(instance.Instance, 'check_create_complete')\
.return_value = True
self.patchobject(stack_resource.StackResource,
'check_update_complete').return_value = True
self.patchobject(loadbalancer.LoadBalancer, 'handle_update')
self.patchobject(user.User, 'handle_create')
self.patchobject(user.AccessKey, 'handle_create')
self.patchobject(waitc.WaitCondition, 'handle_create')
self.patchobject(signal.SignalResponder, 'handle_create')
def expected_notifs_calls(self, group, adjust,
start_capacity, end_capacity=None,
with_error=None):
stack_arn = self.stack.identifier().arn()
expected = [mock.call(self.ctx,
'orchestration.test_host',
'orchestration.autoscaling.start',
'INFO',
{'state_reason':
'Stack create completed successfully',
'user_id': 'test_username',
'stack_identity': stack_arn,
'tenant_id': 'test_tenant',
'create_at': self.create_at,
'adjustment_type': 'ChangeInCapacity',
'groupname': group.FnGetRefId(),
'capacity': start_capacity,
'adjustment': adjust,
'stack_name': self.stack_name,
'message': 'Start resizing the group %s' %
group.FnGetRefId(),
'state': 'CREATE_COMPLETE'})
]
if with_error:
expected += [mock.call(self.ctx,
'orchestration.test_host',
'orchestration.autoscaling.error',
'ERROR',
{'state_reason':
'Stack create completed successfully',
'user_id': 'test_username',
'stack_identity': stack_arn,
'tenant_id': 'test_tenant',
'create_at': self.create_at,
'adjustment_type': 'ChangeInCapacity',
'groupname': group.FnGetRefId(),
'capacity': start_capacity,
'adjustment': adjust,
'stack_name': self.stack_name,
'message': with_error,
'state': 'CREATE_COMPLETE'})
]
else:
expected += [mock.call(self.ctx,
'orchestration.test_host',
'orchestration.autoscaling.end',
'INFO',
{'state_reason':
'Stack create completed successfully',
'user_id': 'test_username',
'stack_identity': stack_arn,
'tenant_id': 'test_tenant',
'create_at': self.create_at,
'adjustment_type': 'ChangeInCapacity',
'groupname': group.FnGetRefId(),
'capacity': end_capacity,
'adjustment': adjust,
'stack_name': self.stack_name,
'message': 'End resizing the group %s' %
group.FnGetRefId(),
'state': 'CREATE_COMPLETE'})
]
return expected
@utils.stack_delete_after
def test_scale_success(self):
with mock.patch('heat.engine.notification.stack.send'):
with mock.patch('heat.openstack.common.notifier.api.notify') \
as mock_notify:
self.mock_stack_except_for_group()
group = self.create_autoscaling_stack_and_get_group()
expected = self.expected_notifs_calls(group,
adjust=1,
start_capacity=1,
end_capacity=2,
)
group.adjust(1)
self.assertEqual(2, len(group.get_instance_names()))
mock_notify.assert_has_calls(expected)
expected = self.expected_notifs_calls(group,
adjust=-1,
start_capacity=2,
end_capacity=1,
)
group.adjust(-1)
self.assertEqual(1, len(group.get_instance_names()))
mock_notify.assert_has_calls(expected)
@utils.stack_delete_after
def test_scaleup_failure(self):
with mock.patch('heat.engine.notification.stack.send'):
with mock.patch('heat.openstack.common.notifier.api.notify') \
as mock_notify:
self.mock_stack_except_for_group()
group = self.create_autoscaling_stack_and_get_group()
err_message = 'Boooom'
m_as = self.patchobject(autoscaling.AutoScalingGroup, 'resize')
m_as.side_effect = exception.Error(err_message)
expected = self.expected_notifs_calls(group,
adjust=2,
start_capacity=1,
with_error=err_message,
)
self.assertRaises(exception.Error, group.adjust, 2)
self.assertEqual(1, len(group.get_instance_names()))
mock_notify.assert_has_calls(expected)