ASG scaling account for cooldown timestamp & in-progress
There are cases where it takes a long time to create a new resource as requested by the scaling operation on an ASG resource, for instance, a nova server creation followed by a complex SoftwareDeployment. During this process, additional alarms may come in but failed to be blocked by the current cooldown checking mechanism because the very first timestamp has yet to be generated. This is leading to unexpected size adjustment to the ASG. This patch augments the existing cooldown checking mechanism with a scaling-in-progress test so that additional alarms arriving during the very first scaling operation will be ignored. Change-Id: Ib8aa83eed366df7097c9cbb9247eca866ae4b620 Closes-Bug: #1375156
This commit is contained in:
parent
7dcb065b55
commit
b76881b8bc
|
@ -354,8 +354,9 @@ class AutoScalingGroup(instgrp.InstanceGroup, cooldown.CooldownMixin):
|
|||
'group': notif['groupname']},
|
||||
})
|
||||
notification.send(**notif)
|
||||
|
||||
self._cooldown_timestamp("%s : %s" % (adjustment_type, adjustment))
|
||||
finally:
|
||||
self._cooldown_timestamp("%s : %s" % (adjustment_type,
|
||||
adjustment))
|
||||
|
||||
def _tags(self):
|
||||
"""Add Identifing Tags to all servers in the group.
|
||||
|
|
|
@ -170,23 +170,27 @@ class AutoScalingPolicy(signal_responder.SignalResponder,
|
|||
|
||||
asgn_id = self.properties[self.AUTO_SCALING_GROUP_NAME]
|
||||
group = self.stack.resource_by_refid(asgn_id)
|
||||
if group is None:
|
||||
raise exception.NotFound(_('Alarm %(alarm)s could not find '
|
||||
'scaling group named "%(group)s"') % {
|
||||
'alarm': self.name,
|
||||
'group': asgn_id})
|
||||
try:
|
||||
if group is None:
|
||||
raise exception.NotFound(_('Alarm %(alarm)s could not find '
|
||||
'scaling group named "%(group)s"'
|
||||
) % {'alarm': self.name,
|
||||
'group': asgn_id})
|
||||
|
||||
LOG.info(_LI('%(name)s Alarm, adjusting Group %(group)s with id '
|
||||
'%(asgn_id)s by %(filter)s'),
|
||||
{'name': self.name, 'group': group.name, 'asgn_id': asgn_id,
|
||||
'filter': self.properties[self.SCALING_ADJUSTMENT]})
|
||||
adjustment_type = self._get_adjustement_type()
|
||||
group.adjust(self.properties[self.SCALING_ADJUSTMENT], adjustment_type,
|
||||
self.properties[self.MIN_ADJUSTMENT_STEP])
|
||||
LOG.info(_LI('%(name)s Alarm, adjusting Group %(group)s with id '
|
||||
'%(asgn_id)s by %(filter)s'),
|
||||
{'name': self.name, 'group': group.name,
|
||||
'asgn_id': asgn_id,
|
||||
'filter': self.properties[self.SCALING_ADJUSTMENT]})
|
||||
adjustment_type = self._get_adjustement_type()
|
||||
group.adjust(self.properties[self.SCALING_ADJUSTMENT],
|
||||
adjustment_type,
|
||||
self.properties[self.MIN_ADJUSTMENT_STEP])
|
||||
|
||||
self._cooldown_timestamp("%s : %s" %
|
||||
(self.properties[self.ADJUSTMENT_TYPE],
|
||||
self.properties[self.SCALING_ADJUSTMENT]))
|
||||
finally:
|
||||
self._cooldown_timestamp("%s : %s" % (
|
||||
self.properties[self.ADJUSTMENT_TYPE],
|
||||
self.properties[self.SCALING_ADJUSTMENT]))
|
||||
|
||||
def _resolve_attribute(self, name):
|
||||
if name == self.ALARM_URL:
|
||||
|
|
|
@ -18,7 +18,8 @@ import six
|
|||
class CooldownMixin(object):
|
||||
'''
|
||||
Utility class to encapsulate Cooldown related logic which is shared
|
||||
between AutoScalingGroup and ScalingPolicy
|
||||
between AutoScalingGroup and ScalingPolicy. This logic includes both
|
||||
cooldown timestamp comparing and scaling in progress checking.
|
||||
'''
|
||||
def _cooldown_inprogress(self):
|
||||
inprogress = False
|
||||
|
@ -30,16 +31,33 @@ class CooldownMixin(object):
|
|||
cooldown = 0
|
||||
|
||||
metadata = self.metadata_get()
|
||||
if metadata and cooldown != 0:
|
||||
last_adjust = next(six.iterkeys(metadata))
|
||||
if metadata.get('scaling_in_progress'):
|
||||
return True
|
||||
|
||||
if 'cooldown' not in metadata:
|
||||
# Note: this is for supporting old version cooldown checking
|
||||
if metadata and cooldown != 0:
|
||||
last_adjust = next(six.iterkeys(metadata))
|
||||
if not timeutils.is_older_than(last_adjust, cooldown):
|
||||
inprogress = True
|
||||
elif cooldown != 0:
|
||||
last_adjust = next(six.iterkeys(metadata['cooldown']))
|
||||
if not timeutils.is_older_than(last_adjust, cooldown):
|
||||
inprogress = True
|
||||
|
||||
if not inprogress:
|
||||
metadata['scaling_in_progress'] = True
|
||||
self.metadata_set(metadata)
|
||||
|
||||
return inprogress
|
||||
|
||||
def _cooldown_timestamp(self, reason):
|
||||
# Save resource metadata with a timestamp and reason
|
||||
# Save cooldown timestamp into metadata and clean the
|
||||
# scaling_in_progress state.
|
||||
# If we wanted to implement the AutoScaling API like AWS does,
|
||||
# we could maintain event history here, but since we only need
|
||||
# the latest event for cooldown, just store that for now
|
||||
metadata = {timeutils.utcnow().isoformat(): reason}
|
||||
metadata = self.metadata_get()
|
||||
metadata['cooldown'] = {timeutils.utcnow().isoformat(): reason}
|
||||
metadata['scaling_in_progress'] = False
|
||||
self.metadata_set(metadata)
|
||||
|
|
|
@ -247,6 +247,7 @@ class TestGroupAdjust(common.HeatTestCase):
|
|||
notify = self.patch('heat.engine.notification.autoscaling.send')
|
||||
self.patchobject(self.group, '_cooldown_inprogress',
|
||||
return_value=False)
|
||||
self.patchobject(self.group, '_cooldown_timestamp')
|
||||
self.assertRaises(ValueError, self.group.adjust, 1)
|
||||
|
||||
expected_notifies = [
|
||||
|
@ -276,6 +277,7 @@ class TestGroupAdjust(common.HeatTestCase):
|
|||
notify = self.patch('heat.engine.notification.autoscaling.send')
|
||||
self.patchobject(self.group, '_cooldown_inprogress',
|
||||
return_value=False)
|
||||
self.patchobject(self.group, '_cooldown_timestamp')
|
||||
|
||||
self.assertRaises(ValueError, self.group.adjust,
|
||||
5, adjustment_type='ExactCapacity')
|
||||
|
|
|
@ -130,23 +130,38 @@ class TestCooldownMixin(common.HeatTestCase):
|
|||
self.assertEqual((rsrc.CREATE, rsrc.COMPLETE), rsrc.state)
|
||||
return rsrc
|
||||
|
||||
def test_is_in_progress(self):
|
||||
def test_cooldown_is_in_progress_toosoon(self):
|
||||
t = template_format.parse(as_template)
|
||||
stack = utils.parse_stack(t, params=as_params)
|
||||
pol = self.create_scaling_policy(t, stack, 'my-policy')
|
||||
|
||||
now = timeutils.utcnow()
|
||||
previous_meta = {now.isoformat(): 'ChangeInCapacity : 1'}
|
||||
previous_meta = {'cooldown': {
|
||||
now.isoformat(): 'ChangeInCapacity : 1'}}
|
||||
self.patchobject(pol, 'metadata_get', return_value=previous_meta)
|
||||
self.assertTrue(pol._cooldown_inprogress())
|
||||
|
||||
def test_not_in_progress(self):
|
||||
def test_cooldown_is_in_progress_scaling_unfinished(self):
|
||||
t = template_format.parse(as_template)
|
||||
stack = utils.parse_stack(t, params=as_params)
|
||||
pol = self.create_scaling_policy(t, stack, 'my-policy')
|
||||
|
||||
previous_meta = {'scaling_in_progress': True}
|
||||
self.patchobject(pol, 'metadata_get', return_value=previous_meta)
|
||||
self.assertTrue(pol._cooldown_inprogress())
|
||||
|
||||
def test_cooldown_not_in_progress(self):
|
||||
t = template_format.parse(as_template)
|
||||
stack = utils.parse_stack(t, params=as_params)
|
||||
pol = self.create_scaling_policy(t, stack, 'my-policy')
|
||||
|
||||
awhile_ago = timeutils.utcnow() - datetime.timedelta(seconds=100)
|
||||
previous_meta = {awhile_ago.isoformat(): 'ChangeInCapacity : 1'}
|
||||
previous_meta = {
|
||||
'cooldown': {
|
||||
awhile_ago.isoformat(): 'ChangeInCapacity : 1'
|
||||
},
|
||||
'scaling_in_progress': False
|
||||
}
|
||||
self.patchobject(pol, 'metadata_get', return_value=previous_meta)
|
||||
self.assertFalse(pol._cooldown_inprogress())
|
||||
|
||||
|
@ -161,7 +176,8 @@ class TestCooldownMixin(common.HeatTestCase):
|
|||
pol = self.create_scaling_policy(t, stack, 'my-policy')
|
||||
|
||||
now = timeutils.utcnow()
|
||||
previous_meta = {now.isoformat(): 'ChangeInCapacity : 1'}
|
||||
previous_meta = {'cooldown': {
|
||||
now.isoformat(): 'ChangeInCapacity : 1'}}
|
||||
self.patchobject(pol, 'metadata_get', return_value=previous_meta)
|
||||
self.assertFalse(pol._cooldown_inprogress())
|
||||
|
||||
|
@ -177,7 +193,8 @@ class TestCooldownMixin(common.HeatTestCase):
|
|||
pol = self.create_scaling_policy(t, stack, 'my-policy')
|
||||
|
||||
now = timeutils.utcnow()
|
||||
previous_meta = {now.isoformat(): 'ChangeInCapacity : 1'}
|
||||
previous_meta = {'cooldown': {
|
||||
now.isoformat(): 'ChangeInCapacity : 1'}}
|
||||
self.patchobject(pol, 'metadata_get', return_value=previous_meta)
|
||||
self.assertFalse(pol._cooldown_inprogress())
|
||||
|
||||
|
@ -191,7 +208,9 @@ class TestCooldownMixin(common.HeatTestCase):
|
|||
meta_set = self.patchobject(pol, 'metadata_set')
|
||||
self.patchobject(timeutils, 'utcnow', return_value=nowish)
|
||||
pol._cooldown_timestamp(reason)
|
||||
meta_set.assert_called_once_with({nowish.isoformat(): reason})
|
||||
meta_set.assert_called_once_with(
|
||||
{'cooldown': {nowish.isoformat(): reason},
|
||||
'scaling_in_progress': False})
|
||||
|
||||
|
||||
class ScalingPolicyAttrTest(common.HeatTestCase):
|
||||
|
|
|
@ -434,6 +434,7 @@ class TestGroupAdjust(common.HeatTestCase):
|
|||
notify = self.patch('heat.engine.notification.autoscaling.send')
|
||||
self.patchobject(self.group, '_cooldown_inprogress',
|
||||
return_value=False)
|
||||
self.patchobject(self.group, '_cooldown_timestamp')
|
||||
self.assertRaises(ValueError, self.group.adjust, 1)
|
||||
|
||||
expected_notifies = [
|
||||
|
@ -463,6 +464,7 @@ class TestGroupAdjust(common.HeatTestCase):
|
|||
notify = self.patch('heat.engine.notification.autoscaling.send')
|
||||
self.patchobject(self.group, '_cooldown_inprogress',
|
||||
return_value=False)
|
||||
self.patchobject(self.group, '_cooldown_timestamp')
|
||||
|
||||
self.assertRaises(ValueError, self.group.adjust,
|
||||
5, adjustment_type='ExactCapacity')
|
||||
|
|
|
@ -135,23 +135,38 @@ class TestCooldownMixin(common.HeatTestCase):
|
|||
self.assertEqual((rsrc.CREATE, rsrc.COMPLETE), rsrc.state)
|
||||
return rsrc
|
||||
|
||||
def test_is_in_progress(self):
|
||||
def test_cooldown_is_in_progress_toosoon(self):
|
||||
t = template_format.parse(as_template)
|
||||
stack = utils.parse_stack(t, params=as_params)
|
||||
pol = self.create_scaling_policy(t, stack, 'WebServerScaleUpPolicy')
|
||||
|
||||
now = timeutils.utcnow()
|
||||
previous_meta = {now.isoformat(): 'ChangeInCapacity : 1'}
|
||||
previous_meta = {'cooldown': {
|
||||
now.isoformat(): 'ChangeInCapacity : 1'}}
|
||||
self.patchobject(pol, 'metadata_get', return_value=previous_meta)
|
||||
self.assertTrue(pol._cooldown_inprogress())
|
||||
|
||||
def test_not_in_progress(self):
|
||||
def test_cooldown_is_in_progress_scaling_unfinished(self):
|
||||
t = template_format.parse(as_template)
|
||||
stack = utils.parse_stack(t, params=as_params)
|
||||
pol = self.create_scaling_policy(t, stack, 'WebServerScaleUpPolicy')
|
||||
|
||||
previous_meta = {'scaling_in_progress': True}
|
||||
self.patchobject(pol, 'metadata_get', return_value=previous_meta)
|
||||
self.assertTrue(pol._cooldown_inprogress())
|
||||
|
||||
def test_cooldown_not_in_progress(self):
|
||||
t = template_format.parse(as_template)
|
||||
stack = utils.parse_stack(t, params=as_params)
|
||||
pol = self.create_scaling_policy(t, stack, 'WebServerScaleUpPolicy')
|
||||
|
||||
awhile_ago = timeutils.utcnow() - datetime.timedelta(seconds=100)
|
||||
previous_meta = {awhile_ago.isoformat(): 'ChangeInCapacity : 1'}
|
||||
previous_meta = {
|
||||
'cooldown': {
|
||||
awhile_ago.isoformat(): 'ChangeInCapacity : 1'
|
||||
},
|
||||
'scaling_in_progress': False
|
||||
}
|
||||
self.patchobject(pol, 'metadata_get', return_value=previous_meta)
|
||||
self.assertFalse(pol._cooldown_inprogress())
|
||||
|
||||
|
@ -196,7 +211,9 @@ class TestCooldownMixin(common.HeatTestCase):
|
|||
meta_set = self.patchobject(pol, 'metadata_set')
|
||||
self.patchobject(timeutils, 'utcnow', return_value=nowish)
|
||||
pol._cooldown_timestamp(reason)
|
||||
meta_set.assert_called_once_with({nowish.isoformat(): reason})
|
||||
meta_set.assert_called_once_with(
|
||||
{'cooldown': {nowish.isoformat(): reason},
|
||||
'scaling_in_progress': False})
|
||||
|
||||
|
||||
class ScalingPolicyAttrTest(common.HeatTestCase):
|
||||
|
|
Loading…
Reference in New Issue