ASG scaling account for cooldown timestamp & in-progress

There are cases where it takes a long time to create a new resource as
requested by the scaling operation on an ASG resource, for instance, a
nova server creation followed by a complex SoftwareDeployment. During
this process, additional alarms may come in but failed to be blocked
by the current cooldown checking mechanism because the very first
timestamp has yet to be generated. This is leading to unexpected size
adjustment to the ASG.

This patch augments the existing cooldown checking mechanism with a
scaling-in-progress test so that additional alarms arriving during
the very first scaling operation will be ignored.

Change-Id: Ib8aa83eed366df7097c9cbb9247eca866ae4b620
Closes-Bug: #1375156
This commit is contained in:
yanyanhu 2015-05-29 01:43:03 -04:00
parent 7dcb065b55
commit b76881b8bc
7 changed files with 97 additions and 34 deletions

View File

@ -354,8 +354,9 @@ class AutoScalingGroup(instgrp.InstanceGroup, cooldown.CooldownMixin):
'group': notif['groupname']},
})
notification.send(**notif)
self._cooldown_timestamp("%s : %s" % (adjustment_type, adjustment))
finally:
self._cooldown_timestamp("%s : %s" % (adjustment_type,
adjustment))
def _tags(self):
"""Add Identifing Tags to all servers in the group.

View File

@ -170,23 +170,27 @@ class AutoScalingPolicy(signal_responder.SignalResponder,
asgn_id = self.properties[self.AUTO_SCALING_GROUP_NAME]
group = self.stack.resource_by_refid(asgn_id)
if group is None:
raise exception.NotFound(_('Alarm %(alarm)s could not find '
'scaling group named "%(group)s"') % {
'alarm': self.name,
'group': asgn_id})
try:
if group is None:
raise exception.NotFound(_('Alarm %(alarm)s could not find '
'scaling group named "%(group)s"'
) % {'alarm': self.name,
'group': asgn_id})
LOG.info(_LI('%(name)s Alarm, adjusting Group %(group)s with id '
'%(asgn_id)s by %(filter)s'),
{'name': self.name, 'group': group.name, 'asgn_id': asgn_id,
'filter': self.properties[self.SCALING_ADJUSTMENT]})
adjustment_type = self._get_adjustement_type()
group.adjust(self.properties[self.SCALING_ADJUSTMENT], adjustment_type,
self.properties[self.MIN_ADJUSTMENT_STEP])
LOG.info(_LI('%(name)s Alarm, adjusting Group %(group)s with id '
'%(asgn_id)s by %(filter)s'),
{'name': self.name, 'group': group.name,
'asgn_id': asgn_id,
'filter': self.properties[self.SCALING_ADJUSTMENT]})
adjustment_type = self._get_adjustement_type()
group.adjust(self.properties[self.SCALING_ADJUSTMENT],
adjustment_type,
self.properties[self.MIN_ADJUSTMENT_STEP])
self._cooldown_timestamp("%s : %s" %
(self.properties[self.ADJUSTMENT_TYPE],
self.properties[self.SCALING_ADJUSTMENT]))
finally:
self._cooldown_timestamp("%s : %s" % (
self.properties[self.ADJUSTMENT_TYPE],
self.properties[self.SCALING_ADJUSTMENT]))
def _resolve_attribute(self, name):
if name == self.ALARM_URL:

View File

@ -18,7 +18,8 @@ import six
class CooldownMixin(object):
'''
Utility class to encapsulate Cooldown related logic which is shared
between AutoScalingGroup and ScalingPolicy
between AutoScalingGroup and ScalingPolicy. This logic includes both
cooldown timestamp comparing and scaling in progress checking.
'''
def _cooldown_inprogress(self):
inprogress = False
@ -30,16 +31,33 @@ class CooldownMixin(object):
cooldown = 0
metadata = self.metadata_get()
if metadata and cooldown != 0:
last_adjust = next(six.iterkeys(metadata))
if metadata.get('scaling_in_progress'):
return True
if 'cooldown' not in metadata:
# Note: this is for supporting old version cooldown checking
if metadata and cooldown != 0:
last_adjust = next(six.iterkeys(metadata))
if not timeutils.is_older_than(last_adjust, cooldown):
inprogress = True
elif cooldown != 0:
last_adjust = next(six.iterkeys(metadata['cooldown']))
if not timeutils.is_older_than(last_adjust, cooldown):
inprogress = True
if not inprogress:
metadata['scaling_in_progress'] = True
self.metadata_set(metadata)
return inprogress
def _cooldown_timestamp(self, reason):
# Save resource metadata with a timestamp and reason
# Save cooldown timestamp into metadata and clean the
# scaling_in_progress state.
# If we wanted to implement the AutoScaling API like AWS does,
# we could maintain event history here, but since we only need
# the latest event for cooldown, just store that for now
metadata = {timeutils.utcnow().isoformat(): reason}
metadata = self.metadata_get()
metadata['cooldown'] = {timeutils.utcnow().isoformat(): reason}
metadata['scaling_in_progress'] = False
self.metadata_set(metadata)

View File

@ -247,6 +247,7 @@ class TestGroupAdjust(common.HeatTestCase):
notify = self.patch('heat.engine.notification.autoscaling.send')
self.patchobject(self.group, '_cooldown_inprogress',
return_value=False)
self.patchobject(self.group, '_cooldown_timestamp')
self.assertRaises(ValueError, self.group.adjust, 1)
expected_notifies = [
@ -276,6 +277,7 @@ class TestGroupAdjust(common.HeatTestCase):
notify = self.patch('heat.engine.notification.autoscaling.send')
self.patchobject(self.group, '_cooldown_inprogress',
return_value=False)
self.patchobject(self.group, '_cooldown_timestamp')
self.assertRaises(ValueError, self.group.adjust,
5, adjustment_type='ExactCapacity')

View File

@ -130,23 +130,38 @@ class TestCooldownMixin(common.HeatTestCase):
self.assertEqual((rsrc.CREATE, rsrc.COMPLETE), rsrc.state)
return rsrc
def test_is_in_progress(self):
def test_cooldown_is_in_progress_toosoon(self):
t = template_format.parse(as_template)
stack = utils.parse_stack(t, params=as_params)
pol = self.create_scaling_policy(t, stack, 'my-policy')
now = timeutils.utcnow()
previous_meta = {now.isoformat(): 'ChangeInCapacity : 1'}
previous_meta = {'cooldown': {
now.isoformat(): 'ChangeInCapacity : 1'}}
self.patchobject(pol, 'metadata_get', return_value=previous_meta)
self.assertTrue(pol._cooldown_inprogress())
def test_not_in_progress(self):
def test_cooldown_is_in_progress_scaling_unfinished(self):
t = template_format.parse(as_template)
stack = utils.parse_stack(t, params=as_params)
pol = self.create_scaling_policy(t, stack, 'my-policy')
previous_meta = {'scaling_in_progress': True}
self.patchobject(pol, 'metadata_get', return_value=previous_meta)
self.assertTrue(pol._cooldown_inprogress())
def test_cooldown_not_in_progress(self):
t = template_format.parse(as_template)
stack = utils.parse_stack(t, params=as_params)
pol = self.create_scaling_policy(t, stack, 'my-policy')
awhile_ago = timeutils.utcnow() - datetime.timedelta(seconds=100)
previous_meta = {awhile_ago.isoformat(): 'ChangeInCapacity : 1'}
previous_meta = {
'cooldown': {
awhile_ago.isoformat(): 'ChangeInCapacity : 1'
},
'scaling_in_progress': False
}
self.patchobject(pol, 'metadata_get', return_value=previous_meta)
self.assertFalse(pol._cooldown_inprogress())
@ -161,7 +176,8 @@ class TestCooldownMixin(common.HeatTestCase):
pol = self.create_scaling_policy(t, stack, 'my-policy')
now = timeutils.utcnow()
previous_meta = {now.isoformat(): 'ChangeInCapacity : 1'}
previous_meta = {'cooldown': {
now.isoformat(): 'ChangeInCapacity : 1'}}
self.patchobject(pol, 'metadata_get', return_value=previous_meta)
self.assertFalse(pol._cooldown_inprogress())
@ -177,7 +193,8 @@ class TestCooldownMixin(common.HeatTestCase):
pol = self.create_scaling_policy(t, stack, 'my-policy')
now = timeutils.utcnow()
previous_meta = {now.isoformat(): 'ChangeInCapacity : 1'}
previous_meta = {'cooldown': {
now.isoformat(): 'ChangeInCapacity : 1'}}
self.patchobject(pol, 'metadata_get', return_value=previous_meta)
self.assertFalse(pol._cooldown_inprogress())
@ -191,7 +208,9 @@ class TestCooldownMixin(common.HeatTestCase):
meta_set = self.patchobject(pol, 'metadata_set')
self.patchobject(timeutils, 'utcnow', return_value=nowish)
pol._cooldown_timestamp(reason)
meta_set.assert_called_once_with({nowish.isoformat(): reason})
meta_set.assert_called_once_with(
{'cooldown': {nowish.isoformat(): reason},
'scaling_in_progress': False})
class ScalingPolicyAttrTest(common.HeatTestCase):

View File

@ -434,6 +434,7 @@ class TestGroupAdjust(common.HeatTestCase):
notify = self.patch('heat.engine.notification.autoscaling.send')
self.patchobject(self.group, '_cooldown_inprogress',
return_value=False)
self.patchobject(self.group, '_cooldown_timestamp')
self.assertRaises(ValueError, self.group.adjust, 1)
expected_notifies = [
@ -463,6 +464,7 @@ class TestGroupAdjust(common.HeatTestCase):
notify = self.patch('heat.engine.notification.autoscaling.send')
self.patchobject(self.group, '_cooldown_inprogress',
return_value=False)
self.patchobject(self.group, '_cooldown_timestamp')
self.assertRaises(ValueError, self.group.adjust,
5, adjustment_type='ExactCapacity')

View File

@ -135,23 +135,38 @@ class TestCooldownMixin(common.HeatTestCase):
self.assertEqual((rsrc.CREATE, rsrc.COMPLETE), rsrc.state)
return rsrc
def test_is_in_progress(self):
def test_cooldown_is_in_progress_toosoon(self):
t = template_format.parse(as_template)
stack = utils.parse_stack(t, params=as_params)
pol = self.create_scaling_policy(t, stack, 'WebServerScaleUpPolicy')
now = timeutils.utcnow()
previous_meta = {now.isoformat(): 'ChangeInCapacity : 1'}
previous_meta = {'cooldown': {
now.isoformat(): 'ChangeInCapacity : 1'}}
self.patchobject(pol, 'metadata_get', return_value=previous_meta)
self.assertTrue(pol._cooldown_inprogress())
def test_not_in_progress(self):
def test_cooldown_is_in_progress_scaling_unfinished(self):
t = template_format.parse(as_template)
stack = utils.parse_stack(t, params=as_params)
pol = self.create_scaling_policy(t, stack, 'WebServerScaleUpPolicy')
previous_meta = {'scaling_in_progress': True}
self.patchobject(pol, 'metadata_get', return_value=previous_meta)
self.assertTrue(pol._cooldown_inprogress())
def test_cooldown_not_in_progress(self):
t = template_format.parse(as_template)
stack = utils.parse_stack(t, params=as_params)
pol = self.create_scaling_policy(t, stack, 'WebServerScaleUpPolicy')
awhile_ago = timeutils.utcnow() - datetime.timedelta(seconds=100)
previous_meta = {awhile_ago.isoformat(): 'ChangeInCapacity : 1'}
previous_meta = {
'cooldown': {
awhile_ago.isoformat(): 'ChangeInCapacity : 1'
},
'scaling_in_progress': False
}
self.patchobject(pol, 'metadata_get', return_value=previous_meta)
self.assertFalse(pol._cooldown_inprogress())
@ -196,7 +211,9 @@ class TestCooldownMixin(common.HeatTestCase):
meta_set = self.patchobject(pol, 'metadata_set')
self.patchobject(timeutils, 'utcnow', return_value=nowish)
pol._cooldown_timestamp(reason)
meta_set.assert_called_once_with({nowish.isoformat(): reason})
meta_set.assert_called_once_with(
{'cooldown': {nowish.isoformat(): reason},
'scaling_in_progress': False})
class ScalingPolicyAttrTest(common.HeatTestCase):