openstack
/
heat
Code Issues Proposed changes

Make sure failures in groups (autoscaling & static) are raised. 

Fixes bug 1104617

Change-Id: Iebd2b0c6a4adf8c252db31348e4dc241c0479765
This commit is contained in:
Angus Salkeld 2013-01-28 21:57:27 +11:00
parent 0eff406e45
commit 6d32597f06
4 changed files with 85 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
heat/common/exception.py
View File

@ -232,3 +232,7 @@ class PhysicalResourceNotFound(OpenstackException):
class WatchRuleNotFound(OpenstackException):
message = _("The Watch Rule (%(watch_name)s) could not be found.")
class NestedResourceFailure(OpenstackException):
message = _("%(message)s")

28
heat/engine/resources/autoscaling.py
View File

@ -13,8 +13,9 @@
# License for the specific language governing permissions and limitations
# under the License.
from heat.engine import resource
from heat.common import exception
from heat.engine.resources import instance
from heat.engine import resource
from heat.openstack.common import log as logging
from heat.openstack.common import timeutils
@ -75,7 +76,7 @@ class InstanceGroup(resource.Resource):
# resource_id is a list of resources
def handle_create(self):
self.resize(int(self.properties['Size']))
self.resize(int(self.properties['Size']), raise_on_error=True)
def handle_update(self):
# TODO(asalkeld) if the only thing that has changed is the size then
@ -108,9 +109,13 @@ class InstanceGroup(resource.Resource):
for victim in inst_list:
logger.debug('handle_delete %s' % victim)
inst = self._make_instance(victim)
inst.destroy()
error_str = inst.destroy()
if error_str is not None:
# try suck out the grouped resouces failure reason
# and re-raise
raise exception.NestedResourceFailure(message=error_str)
def resize(self, new_capacity):
def resize(self, new_capacity, raise_on_error=False):
inst_list = []
if self.resource_id is not None:
inst_list = sorted(self.resource_id.split(','))
@ -129,7 +134,12 @@ class InstanceGroup(resource.Resource):
inst = self._make_instance(name)
inst_list.append(name)
self.resource_id_set(','.join(inst_list))
inst.create()
logger.info('creating inst')
error_str = inst.create()
if raise_on_error and error_str is not None:
# try suck out the grouped resouces failure reason
# and re-raise
raise exception.NestedResourceFailure(message=error_str)
else:
# shrink (kill largest numbered first)
del_list = inst_list[new_capacity:]
@ -192,12 +202,14 @@ class AutoScalingGroup(InstanceGroup, CooldownMixin):
else:
num_to_create = int(self.properties['MinSize'])
self.adjust(num_to_create, adjustment_type='ExactCapacity')
self.adjust(num_to_create, adjustment_type='ExactCapacity',
raise_on_error=True)
def handle_update(self):
return self.UPDATE_REPLACE
def adjust(self, adjustment, adjustment_type='ChangeInCapacity'):
def adjust(self, adjustment, adjustment_type='ChangeInCapacity',
raise_on_error=False):
if self._cooldown_inprogress():
logger.info("%s NOT performing scaling adjustment, cooldown %s" %
(self.name, self.properties['Cooldown']))
@ -227,7 +239,7 @@ class AutoScalingGroup(InstanceGroup, CooldownMixin):
logger.debug('no change in capacity %d' % capacity)
return
self.resize(new_capacity)
self.resize(new_capacity, raise_on_error=raise_on_error)
self._cooldown_timestamp("%s : %s" % (adjustment_type, adjustment))

32
heat/tests/test_autoscaling.py
View File

@ -25,6 +25,7 @@ from heat.common import context
from heat.common import template_format
from heat.engine.resources import autoscaling as asc
from heat.engine.resources import loadbalancer
from heat.engine.resources import instance
from heat.engine import parser
from heat.engine.resource import Metadata
from heat.openstack.common import timeutils
@ -73,12 +74,18 @@ class AutoScalingTest(unittest.TestCase):
resource = asc.ScalingPolicy(resource_name,
t['Resources'][resource_name],
stack)
self.assertEqual(None, resource.validate())
self.assertEqual(None, resource.create())
self.assertEqual(asc.ScalingPolicy.CREATE_COMPLETE,
resource.state)
return resource
def _stub_create(self, num):
self.m.StubOutWithMock(instance.Instance, 'create')
for x in range(num):
instance.Instance.create().AndReturn(None)
def _stub_lb_reload(self, expected_list, unset=True):
if unset:
self.m.VerifyAll()
@ -107,6 +114,7 @@ class AutoScalingTest(unittest.TestCase):
self._stub_lb_reload(['WebServerGroup-0'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 1')
self._stub_create(1)
self.m.ReplayAll()
resource = self.create_scaling_group(t, stack, 'WebServerGroup')
@ -129,6 +137,7 @@ class AutoScalingTest(unittest.TestCase):
'WebServerGroup-2'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 3')
self._stub_create(3)
self.m.ReplayAll()
resource = self.create_scaling_group(t, stack, 'WebServerGroup')
self.assertEqual('WebServerGroup-0,WebServerGroup-1,WebServerGroup-2',
@ -145,6 +154,7 @@ class AutoScalingTest(unittest.TestCase):
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1',
'WebServerGroup-2'])
self._stub_meta_expected(now, 'ChangeInCapacity : 2')
self._stub_create(2)
self.m.ReplayAll()
resource.adjust(2)
self.assertEqual('WebServerGroup-0,WebServerGroup-1,WebServerGroup-2',
@ -169,6 +179,7 @@ class AutoScalingTest(unittest.TestCase):
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 2')
self._stub_create(2)
self.m.ReplayAll()
resource = self.create_scaling_group(t, stack, 'WebServerGroup')
stack.resources['WebServerGroup'] = resource
@ -200,6 +211,7 @@ class AutoScalingTest(unittest.TestCase):
properties = t['Resources']['WebServerGroup']['Properties']
properties['DesiredCapacity'] = '2'
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
self._stub_create(2)
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 2')
self.m.ReplayAll()
@ -220,6 +232,7 @@ class AutoScalingTest(unittest.TestCase):
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1',
'WebServerGroup-2'])
self._stub_meta_expected(now, 'PercentChangeInCapacity : 200')
self._stub_create(2)
self.m.ReplayAll()
resource.adjust(200, 'PercentChangeInCapacity')
self.assertEqual('WebServerGroup-0,WebServerGroup-1,WebServerGroup-2',
@ -238,6 +251,7 @@ class AutoScalingTest(unittest.TestCase):
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 2')
self._stub_create(2)
self.m.ReplayAll()
resource = self.create_scaling_group(t, stack, 'WebServerGroup')
stack.resources['WebServerGroup'] = resource
@ -288,6 +302,7 @@ class AutoScalingTest(unittest.TestCase):
properties['DesiredCapacity'] = '2'
properties['Cooldown'] = '60'
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
self._stub_create(2)
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 2')
self.m.ReplayAll()
@ -322,6 +337,7 @@ class AutoScalingTest(unittest.TestCase):
# raise by 200%, should work
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1',
'WebServerGroup-2'], unset=False)
self._stub_create(2)
self._stub_meta_expected(now, 'PercentChangeInCapacity : 200')
self.m.ReplayAll()
resource.adjust(200, 'PercentChangeInCapacity')
@ -341,6 +357,7 @@ class AutoScalingTest(unittest.TestCase):
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 2')
self._stub_create(2)
self.m.ReplayAll()
resource = self.create_scaling_group(t, stack, 'WebServerGroup')
stack.resources['WebServerGroup'] = resource
@ -370,6 +387,7 @@ class AutoScalingTest(unittest.TestCase):
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1',
'WebServerGroup-2'], unset=False)
self._stub_meta_expected(now, 'PercentChangeInCapacity : 200')
self._stub_create(2)
self.m.ReplayAll()
resource.adjust(200, 'PercentChangeInCapacity')
self.assertEqual('WebServerGroup-0,WebServerGroup-1,WebServerGroup-2',
@ -386,6 +404,7 @@ class AutoScalingTest(unittest.TestCase):
self._stub_lb_reload(['WebServerGroup-0'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 1')
self._stub_create(1)
self.m.ReplayAll()
resource = self.create_scaling_group(t, stack, 'WebServerGroup')
stack.resources['WebServerGroup'] = resource
@ -394,6 +413,7 @@ class AutoScalingTest(unittest.TestCase):
# Scale up one
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2)
self._stub_create(1)
self.m.ReplayAll()
up_policy = self.create_scaling_policy(t, stack,
'WebServerScaleUpPolicy')
@ -414,6 +434,7 @@ class AutoScalingTest(unittest.TestCase):
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 2')
self._stub_create(2)
self.m.ReplayAll()
resource = self.create_scaling_group(t, stack, 'WebServerGroup')
stack.resources['WebServerGroup'] = resource
@ -440,6 +461,7 @@ class AutoScalingTest(unittest.TestCase):
self._stub_lb_reload(['WebServerGroup-0'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 1')
self._stub_create(1)
self.m.ReplayAll()
resource = self.create_scaling_group(t, stack, 'WebServerGroup')
stack.resources['WebServerGroup'] = resource
@ -448,6 +470,7 @@ class AutoScalingTest(unittest.TestCase):
# Scale up one
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2)
self._stub_create(1)
self.m.ReplayAll()
up_policy = self.create_scaling_policy(t, stack,
'WebServerScaleUpPolicy')
@ -489,6 +512,7 @@ class AutoScalingTest(unittest.TestCase):
self._stub_lb_reload(['WebServerGroup-0'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 1')
self._stub_create(1)
self.m.ReplayAll()
resource = self.create_scaling_group(t, stack, 'WebServerGroup')
stack.resources['WebServerGroup'] = resource
@ -497,6 +521,7 @@ class AutoScalingTest(unittest.TestCase):
# Scale up one
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2)
self._stub_create(1)
self.m.ReplayAll()
up_policy = self.create_scaling_policy(t, stack,
'WebServerScaleUpPolicy')
@ -520,6 +545,7 @@ class AutoScalingTest(unittest.TestCase):
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1',
'WebServerGroup-2'], unset=False)
self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2)
self._stub_create(1)
self.m.ReplayAll()
up_policy.alarm()
@ -537,6 +563,7 @@ class AutoScalingTest(unittest.TestCase):
self._stub_lb_reload(['WebServerGroup-0'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 1')
self._stub_create(1)
self.m.ReplayAll()
resource = self.create_scaling_group(t, stack, 'WebServerGroup')
stack.resources['WebServerGroup'] = resource
@ -547,6 +574,7 @@ class AutoScalingTest(unittest.TestCase):
properties['Cooldown'] = '0'
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2)
self._stub_create(1)
self.m.ReplayAll()
up_policy = self.create_scaling_policy(t, stack,
'WebServerScaleUpPolicy')
@ -568,6 +596,7 @@ class AutoScalingTest(unittest.TestCase):
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1',
'WebServerGroup-2'], unset=False)
self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2)
self._stub_create(1)
self.m.ReplayAll()
up_policy.alarm()
@ -585,6 +614,7 @@ class AutoScalingTest(unittest.TestCase):
self._stub_lb_reload(['WebServerGroup-0'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ExactCapacity : 1')
self._stub_create(1)
self.m.ReplayAll()
resource = self.create_scaling_group(t, stack, 'WebServerGroup')
stack.resources['WebServerGroup'] = resource
@ -597,6 +627,7 @@ class AutoScalingTest(unittest.TestCase):
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
now = timeutils.utcnow()
self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2)
self._stub_create(1)
self.m.ReplayAll()
up_policy = self.create_scaling_policy(t, stack,
'WebServerScaleUpPolicy')
@ -618,6 +649,7 @@ class AutoScalingTest(unittest.TestCase):
self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1',
'WebServerGroup-2'], unset=False)
self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2)
self._stub_create(1)
self.m.ReplayAll()
up_policy.alarm()

29
heat/tests/test_instance_group.py
View File

@ -20,9 +20,11 @@ import mox
from nose.plugins.attrib import attr
from heat.tests.v1_1 import fakes
from heat.common import context
from heat.common import template_format
from heat.engine.resources import autoscaling as asc
from heat.engine.resources import instance
from heat.engine.resources import loadbalancer
from heat.engine import parser
@ -31,6 +33,7 @@ from heat.engine import parser
@attr(speed='fast')
class InstanceGroupTest(unittest.TestCase):
def setUp(self):
self.fc = fakes.FakeClient()
self.m = mox.Mox()
self.m.StubOutWithMock(loadbalancer.LoadBalancer, 'reload')
@ -58,6 +61,11 @@ class InstanceGroupTest(unittest.TestCase):
return stack
def _stub_create(self, num):
self.m.StubOutWithMock(instance.Instance, 'create')
for x in range(num):
instance.Instance.create().AndReturn(None)
def create_instance_group(self, t, stack, resource_name):
resource = asc.InstanceGroup(resource_name,
t['Resources'][resource_name],
@ -73,6 +81,8 @@ class InstanceGroupTest(unittest.TestCase):
stack = self.parse_stack(t)
# start with min then delete
self._stub_create(1)
self.m.ReplayAll()
resource = self.create_instance_group(t, stack, 'JobServerGroup')
self.assertEqual('JobServerGroup', resource.FnGetRefId())
@ -81,3 +91,22 @@ class InstanceGroupTest(unittest.TestCase):
resource.handle_update())
resource.delete()
def test_missing_image(self):
t = self.load_template()
stack = self.parse_stack(t)
resource = asc.InstanceGroup('JobServerGroup',
t['Resources']['JobServerGroup'],
stack)
self.m.StubOutWithMock(instance.Instance, 'create')
instance.Instance.create().AndReturn('ImageNotFound: bla')
self.m.ReplayAll()
self.assertEqual(resource.create(), 'ImageNotFound: bla')
self.assertEqual(asc.InstanceGroup.CREATE_FAILED, resource.state)
self.m.VerifyAll()