openstack/deb-heat
Code Issues Proposed changes

InstanceGroup to keep FAILED resources in template

Browse Source 
Modify the InstanceGroup (and, by extension, Heat and AWS
AutoscalingGroup) types to give members in a FAILED state the highest
priority for being removed when scaling down or being updated in
a rolling update. Currently, FAILED resources are omitted when building
a new template for the scaling group, so any such resources would never
be replaced by one of the same name. This change will allow for
continuity of naming in the case of a change that doesn't permanently
remove the resource due to scaling down.

Change-Id: I8fe93869dfbe4c4de6c48a65aeef0f71355eaf46
Partially-implements: blueprint mark-unhealthy
This commit is contained in:
Ahmed Elkhouly 2016-01-20 02:06:16 -05:00
parent 8154b1feac
commit 51124ed8c5
3 changed files with 65 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
heat/common/grouputils.py
View File

@ -30,17 +30,20 @@ def get_size(group, include_failed=False):
return 0
def get_members(group):
def get_members(group, include_failed=False):
"""Get a list of member resources managed by the specified group.
Sort the list of instances first by created_time then by name.
If include_failed is set, failed members will be put first in the
list sorted by created_time then by name.
"""
resources = []
if group.nested():
resources = [r for r in six.itervalues(group.nested())
if r.status != r.FAILED]
if include_failed or r.status != r.FAILED]
return sorted(resources, key=lambda r: (r.created_time, r.name))
return sorted(resources,
key=lambda r: (r.status != r.FAILED, r.created_time, r.name))
def get_member_refids(group, exclude=None):
@ -100,10 +103,12 @@ def get_nested_attrs(stack, key, use_indices, *path):
return get_rsrc_id(stack, key, use_indices, *path)
def get_member_definitions(group):
def get_member_definitions(group, include_failed=False):
"""Get member definitions in (name, ResourceDefinition) pair for group.
The List is sorted first by created_time then by name.
If include_failed is set, failed members will be put first in the
List sorted by created_time then by name.
"""
return [(resource.name, resource.t)
for resource in get_members(group)]
for resource in get_members(group, include_failed)]

3
heat/engine/resources/openstack/heat/instance_group.py
View File

@ -249,7 +249,8 @@ class InstanceGroup(stack_resource.StackResource):
Also see heat.scaling.template.member_definitions.
"""
instance_definition = self._get_resource_definition()
old_resources = grouputils.get_member_definitions(self)
old_resources = grouputils.get_member_definitions(self,
include_failed=True)
definitions = template.member_definitions(
old_resources, instance_definition, num_instances, num_replace,
short_id.generate_id)

72
heat/tests/openstack/heat/test_instance_group.py
View File

@ -347,15 +347,9 @@ class LoadbalancerReloadTest(common.HeatTestCase):
{'Instances': ['aaaabbbbcccc']})
class ReplaceTest(common.HeatTestCase):
scenarios = [
('1', dict(min_in_service=0, batch_size=1, updates=2)),
('2', dict(min_in_service=0, batch_size=2, updates=1)),
('3', dict(min_in_service=3, batch_size=1, updates=3)),
('4', dict(min_in_service=3, batch_size=2, updates=2))]
class InstanceGroupWithNestedStack(common.HeatTestCase):
def setUp(self):
super(ReplaceTest, self).setUp()
super(InstanceGroupWithNestedStack, self).setUp()
t = template_format.parse(inline_templates.as_template)
self.stack = utils.parse_stack(t, params=inline_templates.as_params)
lc = self.create_launch_config(t, self.stack)
@ -369,7 +363,6 @@ class ReplaceTest(common.HeatTestCase):
self.group._lb_reload = mock.Mock()
self.group.update_with_template = mock.Mock()
self.group.check_update_complete = mock.Mock()
self.group._nested = self.get_fake_nested_stack()
def create_launch_config(self, t, stack):
self.stub_ImageConstraint_validate()
@ -381,22 +374,34 @@ class ReplaceTest(common.HeatTestCase):
self.assertEqual((rsrc.CREATE, rsrc.COMPLETE), rsrc.state)
return rsrc
def get_fake_nested_stack(self):
nested_t = '''
def get_fake_nested_stack(self, size=1):
tmpl = '''
heat_template_version: 2013-05-23
description: AutoScaling Test
resources:
one:
type: ResourceWithPropsAndAttrs
properties:
Foo: hello
two:
type: ResourceWithPropsAndAttrs
properties:
Foo: fee
'''
resource = '''
r%(i)d:
type: ResourceWithPropsAndAttrs
properties:
Foo: bar%(i)d
'''
resources = '\n'.join([resource % {'i': i + 1} for i in range(size)])
nested_t = tmpl + resources
return utils.parse_stack(template_format.parse(nested_t))
class ReplaceTest(InstanceGroupWithNestedStack):
scenarios = [
('1', dict(min_in_service=0, batch_size=1, updates=2)),
('2', dict(min_in_service=0, batch_size=2, updates=1)),
('3', dict(min_in_service=3, batch_size=1, updates=3)),
('4', dict(min_in_service=3, batch_size=2, updates=2))]
def setUp(self):
super(ReplaceTest, self).setUp()
self.group._nested = self.get_fake_nested_stack(2)
def test_rolling_updates(self):
self.group._replace(self.min_in_service, self.batch_size, 0)
self.assertEqual(self.updates,
@ -405,6 +410,35 @@ class ReplaceTest(common.HeatTestCase):
len(self.group._lb_reload.call_args_list))
class ResizeWithFailedInstancesTest(InstanceGroupWithNestedStack):
scenarios = [
('1', dict(size=3, failed=['r1'], content={'r2', 'r3', 'r4'})),
('2', dict(size=3, failed=['r4'], content={'r1', 'r2', 'r3'})),
('3', dict(size=2, failed=['r1', 'r2'], content={'r3', 'r4'})),
('4', dict(size=2, failed=['r3', 'r4'], content={'r1', 'r2'})),
('5', dict(size=2, failed=['r2', 'r3'], content={'r1', 'r4'})),
('6', dict(size=3, failed=['r2', 'r3'], content={'r1', 'r3', 'r4'}))]
def setUp(self):
super(ResizeWithFailedInstancesTest, self).setUp()
self.group._nested = self.get_fake_nested_stack(4)
self.nested = self.group.nested()
self.group.nested = mock.Mock(return_value=self.nested)
def set_failed_instance(self, instance):
for r in six.itervalues(self.group.nested()):
if r.name == instance:
r.status = "FAILED"
def test_resize(self):
for inst in self.failed:
self.set_failed_instance(inst)
self.group.resize(self.size)
tmpl = self.group.update_with_template.call_args[0][0]
resources = tmpl.resource_definitions(self.group.nested())
self.assertEqual(set(resources.keys()), self.content)
class TestGetBatches(common.HeatTestCase):
scenarios = [