495 lines
21 KiB
Python
495 lines
21 KiB
Python
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import copy
|
|
import json
|
|
|
|
from testtools import matchers
|
|
|
|
from heat_integrationtests.common import test
|
|
|
|
|
|
class InstanceGroupTest(test.HeatIntegrationTest):
|
|
|
|
template = '''
|
|
{
|
|
"AWSTemplateFormatVersion" : "2010-09-09",
|
|
"Description" : "Template to create multiple instances.",
|
|
"Parameters" : {"size": {"Type": "String", "Default": "1"},
|
|
"AZ": {"Type": "String", "Default": "nova"},
|
|
"image": {"Type": "String"},
|
|
"flavor": {"Type": "String"}},
|
|
"Resources": {
|
|
"JobServerGroup": {
|
|
"Type": "OS::Heat::InstanceGroup",
|
|
"Properties": {
|
|
"LaunchConfigurationName" : {"Ref": "JobServerConfig"},
|
|
"Size" : {"Ref": "size"},
|
|
"AvailabilityZones" : [{"Ref": "AZ"}]
|
|
}
|
|
},
|
|
|
|
"JobServerConfig" : {
|
|
"Type" : "AWS::AutoScaling::LaunchConfiguration",
|
|
"Metadata": {"foo": "bar"},
|
|
"Properties": {
|
|
"ImageId" : {"Ref": "image"},
|
|
"InstanceType" : {"Ref": "flavor"},
|
|
"SecurityGroups" : [ "sg-1" ],
|
|
"UserData" : "jsconfig data"
|
|
}
|
|
}
|
|
},
|
|
"Outputs": {
|
|
"InstanceList": {"Value": {
|
|
"Fn::GetAtt": ["JobServerGroup", "InstanceList"]}},
|
|
"JobServerConfigRef": {"Value": {
|
|
"Ref": "JobServerConfig"}}
|
|
}
|
|
}
|
|
'''
|
|
|
|
instance_template = '''
|
|
heat_template_version: 2013-05-23
|
|
parameters:
|
|
ImageId: {type: string}
|
|
InstanceType: {type: string}
|
|
SecurityGroups: {type: comma_delimited_list}
|
|
UserData: {type: string}
|
|
Tags: {type: comma_delimited_list}
|
|
|
|
resources:
|
|
random1:
|
|
type: OS::Heat::RandomString
|
|
properties:
|
|
salt: {get_param: ImageId}
|
|
outputs:
|
|
PublicIp:
|
|
value: {get_attr: [random1, value]}
|
|
'''
|
|
|
|
# This is designed to fail.
|
|
bad_instance_template = '''
|
|
heat_template_version: 2013-05-23
|
|
parameters:
|
|
ImageId: {type: string}
|
|
InstanceType: {type: string}
|
|
SecurityGroups: {type: comma_delimited_list}
|
|
UserData: {type: string}
|
|
Tags: {type: comma_delimited_list}
|
|
|
|
resources:
|
|
random1:
|
|
type: OS::Heat::RandomString
|
|
depends_on: waiter
|
|
ready_poster:
|
|
type: AWS::CloudFormation::WaitConditionHandle
|
|
waiter:
|
|
type: AWS::CloudFormation::WaitCondition
|
|
properties:
|
|
Handle: {Ref: ready_poster}
|
|
Timeout: 1
|
|
outputs:
|
|
PublicIp:
|
|
value: {get_attr: [random1, value]}
|
|
'''
|
|
|
|
def setUp(self):
|
|
super(InstanceGroupTest, self).setUp()
|
|
self.client = self.orchestration_client
|
|
if not self.conf.image_ref:
|
|
raise self.skipException("No image configured to test")
|
|
if not self.conf.minimal_image_ref:
|
|
raise self.skipException("No minimal image configured to test")
|
|
if not self.conf.instance_type:
|
|
raise self.skipException("No flavor configured to test")
|
|
|
|
def assert_instance_count(self, stack, expected_count):
|
|
inst_list = self._stack_output(stack, 'InstanceList')
|
|
self.assertEqual(expected_count, len(inst_list.split(',')))
|
|
|
|
def _assert_instance_state(self, nested_identifier,
|
|
num_complete, num_failed):
|
|
for res in self.client.resources.list(nested_identifier):
|
|
if 'COMPLETE' in res.resource_status:
|
|
num_complete = num_complete - 1
|
|
elif 'FAILED' in res.resource_status:
|
|
num_failed = num_failed - 1
|
|
self.assertEqual(0, num_failed)
|
|
self.assertEqual(0, num_complete)
|
|
|
|
|
|
class InstanceGroupBasicTest(InstanceGroupTest):
|
|
|
|
def test_basic_create_works(self):
|
|
"""Make sure the working case is good.
|
|
Note this combines test_override_aws_ec2_instance into this test as
|
|
well, which is:
|
|
If AWS::EC2::Instance is overridden, InstanceGroup will automatically
|
|
use that overridden resource type.
|
|
"""
|
|
|
|
files = {'provider.yaml': self.instance_template}
|
|
env = {'resource_registry': {'AWS::EC2::Instance': 'provider.yaml'},
|
|
'parameters': {'size': 4,
|
|
'image': self.conf.image_ref,
|
|
'flavor': self.conf.instance_type}}
|
|
stack_identifier = self.stack_create(template=self.template,
|
|
files=files, environment=env)
|
|
initial_resources = {
|
|
'JobServerConfig': 'AWS::AutoScaling::LaunchConfiguration',
|
|
'JobServerGroup': 'OS::Heat::InstanceGroup'}
|
|
self.assertEqual(initial_resources,
|
|
self.list_resources(stack_identifier))
|
|
|
|
stack = self.client.stacks.get(stack_identifier)
|
|
self.assert_instance_count(stack, 4)
|
|
|
|
def test_size_updates_work(self):
|
|
files = {'provider.yaml': self.instance_template}
|
|
env = {'resource_registry': {'AWS::EC2::Instance': 'provider.yaml'},
|
|
'parameters': {'size': 2,
|
|
'image': self.conf.image_ref,
|
|
'flavor': self.conf.instance_type}}
|
|
|
|
stack_identifier = self.stack_create(template=self.template,
|
|
files=files,
|
|
environment=env)
|
|
stack = self.client.stacks.get(stack_identifier)
|
|
self.assert_instance_count(stack, 2)
|
|
|
|
# Increase min size to 5
|
|
env2 = {'resource_registry': {'AWS::EC2::Instance': 'provider.yaml'},
|
|
'parameters': {'size': 5,
|
|
'image': self.conf.image_ref,
|
|
'flavor': self.conf.instance_type}}
|
|
self.update_stack(stack_identifier, self.template,
|
|
environment=env2, files=files)
|
|
stack = self.client.stacks.get(stack_identifier)
|
|
self.assert_instance_count(stack, 5)
|
|
|
|
def test_update_group_replace(self):
|
|
"""Make sure that during a group update the non updatable
|
|
properties cause a replacement.
|
|
"""
|
|
files = {'provider.yaml': self.instance_template}
|
|
env = {'resource_registry':
|
|
{'AWS::EC2::Instance': 'provider.yaml'},
|
|
'parameters': {'size': 1,
|
|
'image': self.conf.image_ref,
|
|
'flavor': self.conf.instance_type}}
|
|
|
|
stack_identifier = self.stack_create(template=self.template,
|
|
files=files,
|
|
environment=env)
|
|
rsrc = self.client.resources.get(stack_identifier, 'JobServerGroup')
|
|
orig_asg_id = rsrc.physical_resource_id
|
|
|
|
env2 = {'resource_registry':
|
|
{'AWS::EC2::Instance': 'provider.yaml'},
|
|
'parameters': {'size': '2',
|
|
'AZ': 'wibble',
|
|
'image': self.conf.image_ref,
|
|
'flavor': self.conf.instance_type}}
|
|
self.update_stack(stack_identifier, self.template,
|
|
environment=env2, files=files)
|
|
|
|
# replacement will cause the resource physical_resource_id to change.
|
|
rsrc = self.client.resources.get(stack_identifier, 'JobServerGroup')
|
|
self.assertNotEqual(orig_asg_id, rsrc.physical_resource_id)
|
|
|
|
def test_create_instance_error_causes_group_error(self):
|
|
"""If a resource in an instance group fails to be created, the instance
|
|
group itself will fail and the broken inner resource will remain.
|
|
"""
|
|
stack_name = self._stack_rand_name()
|
|
files = {'provider.yaml': self.bad_instance_template}
|
|
env = {'resource_registry': {'AWS::EC2::Instance': 'provider.yaml'},
|
|
'parameters': {'size': 2,
|
|
'image': self.conf.image_ref,
|
|
'flavor': self.conf.instance_type}}
|
|
|
|
self.client.stacks.create(
|
|
stack_name=stack_name,
|
|
template=self.template,
|
|
files=files,
|
|
disable_rollback=True,
|
|
parameters={},
|
|
environment=env
|
|
)
|
|
self.addCleanup(self.client.stacks.delete, stack_name)
|
|
stack = self.client.stacks.get(stack_name)
|
|
stack_identifier = '%s/%s' % (stack_name, stack.id)
|
|
self._wait_for_stack_status(stack_identifier, 'CREATE_FAILED')
|
|
initial_resources = {
|
|
'JobServerConfig': 'AWS::AutoScaling::LaunchConfiguration',
|
|
'JobServerGroup': 'OS::Heat::InstanceGroup'}
|
|
self.assertEqual(initial_resources,
|
|
self.list_resources(stack_identifier))
|
|
|
|
nested_ident = self.assert_resource_is_a_stack(stack_identifier,
|
|
'JobServerGroup')
|
|
self._assert_instance_state(nested_ident, 0, 2)
|
|
|
|
def test_update_instance_error_causes_group_error(self):
|
|
"""If a resource in an instance group fails to be created during an
|
|
update, the instance group itself will fail and the broken inner
|
|
resource will remain.
|
|
"""
|
|
files = {'provider.yaml': self.instance_template}
|
|
env = {'resource_registry': {'AWS::EC2::Instance': 'provider.yaml'},
|
|
'parameters': {'size': 2,
|
|
'image': self.conf.image_ref,
|
|
'flavor': self.conf.instance_type}}
|
|
|
|
stack_identifier = self.stack_create(template=self.template,
|
|
files=files,
|
|
environment=env)
|
|
initial_resources = {
|
|
'JobServerConfig': 'AWS::AutoScaling::LaunchConfiguration',
|
|
'JobServerGroup': 'OS::Heat::InstanceGroup'}
|
|
self.assertEqual(initial_resources,
|
|
self.list_resources(stack_identifier))
|
|
|
|
stack = self.client.stacks.get(stack_identifier)
|
|
self.assert_instance_count(stack, 2)
|
|
nested_ident = self.assert_resource_is_a_stack(stack_identifier,
|
|
'JobServerGroup')
|
|
self._assert_instance_state(nested_ident, 2, 0)
|
|
initial_list = [res.resource_name
|
|
for res in self.client.resources.list(nested_ident)]
|
|
|
|
env['parameters']['size'] = 3
|
|
files2 = {'provider.yaml': self.bad_instance_template}
|
|
self.client.stacks.update(
|
|
stack_id=stack_identifier,
|
|
template=self.template,
|
|
files=files2,
|
|
disable_rollback=True,
|
|
parameters={},
|
|
environment=env
|
|
)
|
|
self._wait_for_stack_status(stack_identifier, 'UPDATE_FAILED')
|
|
|
|
nested_ident = self.assert_resource_is_a_stack(stack_identifier,
|
|
'JobServerGroup')
|
|
# assert that there are 3 bad instances
|
|
# 2 resources should be in update failed, and one create failed.
|
|
for res in self.client.resources.list(nested_ident):
|
|
if res.resource_name in initial_list:
|
|
self._wait_for_resource_status(nested_ident,
|
|
res.resource_name,
|
|
'UPDATE_FAILED')
|
|
else:
|
|
self._wait_for_resource_status(nested_ident,
|
|
res.resource_name,
|
|
'CREATE_FAILED')
|
|
|
|
|
|
class InstanceGroupUpdatePolicyTest(InstanceGroupTest):
|
|
|
|
def ig_tmpl_with_updt_policy(self):
|
|
templ = json.loads(copy.deepcopy(self.template))
|
|
up = {"RollingUpdate": {
|
|
"MinInstancesInService": "1",
|
|
"MaxBatchSize": "2",
|
|
"PauseTime": "PT1S"}}
|
|
templ['Resources']['JobServerGroup']['UpdatePolicy'] = up
|
|
return templ
|
|
|
|
def update_instance_group(self, updt_template,
|
|
num_updates_expected_on_updt,
|
|
num_creates_expected_on_updt,
|
|
num_deletes_expected_on_updt,
|
|
update_replace):
|
|
|
|
# setup stack from the initial template
|
|
files = {'provider.yaml': self.instance_template}
|
|
size = 5
|
|
env = {'resource_registry': {'AWS::EC2::Instance': 'provider.yaml'},
|
|
'parameters': {'size': size,
|
|
'image': self.conf.image_ref,
|
|
'flavor': self.conf.instance_type}}
|
|
stack_name = self._stack_rand_name()
|
|
stack_identifier = self.stack_create(
|
|
stack_name=stack_name,
|
|
template=self.ig_tmpl_with_updt_policy(),
|
|
files=files,
|
|
environment=env)
|
|
stack = self.client.stacks.get(stack_identifier)
|
|
nested_ident = self.assert_resource_is_a_stack(stack_identifier,
|
|
'JobServerGroup')
|
|
|
|
# test that physical resource name of launch configuration is used
|
|
conf_name = self._stack_output(stack, 'JobServerConfigRef')
|
|
conf_name_pattern = '%s-JobServerConfig-[a-zA-Z0-9]+$' % stack_name
|
|
self.assertThat(conf_name,
|
|
matchers.MatchesRegex(conf_name_pattern))
|
|
|
|
# test the number of instances created
|
|
self.assert_instance_count(stack, size)
|
|
# saves info from initial list of instances for comparison later
|
|
init_instances = self.client.resources.list(nested_ident)
|
|
init_names = [inst.resource_name for inst in init_instances]
|
|
|
|
# test stack update
|
|
self.update_stack(stack_identifier, updt_template,
|
|
environment=env, files=files)
|
|
updt_stack = self.client.stacks.get(stack_identifier)
|
|
|
|
# test that the launch configuration is replaced
|
|
updt_conf_name = self._stack_output(updt_stack, 'JobServerConfigRef')
|
|
self.assertThat(updt_conf_name,
|
|
matchers.MatchesRegex(conf_name_pattern))
|
|
self.assertNotEqual(conf_name, updt_conf_name)
|
|
|
|
# test that the group size are the same
|
|
updt_instances = self.client.resources.list(nested_ident)
|
|
updt_names = [inst.resource_name for inst in updt_instances]
|
|
self.assertEqual(len(init_names), len(updt_names))
|
|
for res in updt_instances:
|
|
self.assertEqual('UPDATE_COMPLETE', res.resource_status)
|
|
|
|
# test that the appropriate number of instance names are the same
|
|
matched_names = set(updt_names) & set(init_names)
|
|
self.assertEqual(num_updates_expected_on_updt, len(matched_names))
|
|
|
|
# test that the appropriate number of new instances are created
|
|
self.assertEqual(num_creates_expected_on_updt,
|
|
len(set(updt_names) - set(init_names)))
|
|
|
|
# test that the appropriate number of instances are deleted
|
|
self.assertEqual(num_deletes_expected_on_updt,
|
|
len(set(init_names) - set(updt_names)))
|
|
|
|
# test that the older instances are the ones being deleted
|
|
if num_deletes_expected_on_updt > 0:
|
|
deletes_expected = init_names[:num_deletes_expected_on_updt]
|
|
self.assertNotIn(deletes_expected, updt_names)
|
|
|
|
def test_instance_group_update_replace(self):
|
|
"""
|
|
Test simple update replace with no conflict in batch size and
|
|
minimum instances in service.
|
|
"""
|
|
updt_template = self.ig_tmpl_with_updt_policy()
|
|
grp = updt_template['Resources']['JobServerGroup']
|
|
policy = grp['UpdatePolicy']['RollingUpdate']
|
|
policy['MinInstancesInService'] = '1'
|
|
policy['MaxBatchSize'] = '3'
|
|
config = updt_template['Resources']['JobServerConfig']
|
|
config['Properties']['ImageId'] = self.conf.minimal_image_ref
|
|
|
|
self.update_instance_group(updt_template,
|
|
num_updates_expected_on_updt=5,
|
|
num_creates_expected_on_updt=0,
|
|
num_deletes_expected_on_updt=0,
|
|
update_replace=True)
|
|
|
|
def test_instance_group_update_replace_with_adjusted_capacity(self):
|
|
"""
|
|
Test update replace with capacity adjustment due to conflict in
|
|
batch size and minimum instances in service.
|
|
"""
|
|
updt_template = self.ig_tmpl_with_updt_policy()
|
|
grp = updt_template['Resources']['JobServerGroup']
|
|
policy = grp['UpdatePolicy']['RollingUpdate']
|
|
policy['MinInstancesInService'] = '4'
|
|
policy['MaxBatchSize'] = '4'
|
|
config = updt_template['Resources']['JobServerConfig']
|
|
config['Properties']['ImageId'] = self.conf.minimal_image_ref
|
|
|
|
self.update_instance_group(updt_template,
|
|
num_updates_expected_on_updt=2,
|
|
num_creates_expected_on_updt=3,
|
|
num_deletes_expected_on_updt=3,
|
|
update_replace=True)
|
|
|
|
def test_instance_group_update_replace_huge_batch_size(self):
|
|
"""
|
|
Test update replace with a huge batch size.
|
|
"""
|
|
updt_template = self.ig_tmpl_with_updt_policy()
|
|
group = updt_template['Resources']['JobServerGroup']
|
|
policy = group['UpdatePolicy']['RollingUpdate']
|
|
policy['MinInstancesInService'] = '0'
|
|
policy['MaxBatchSize'] = '20'
|
|
config = updt_template['Resources']['JobServerConfig']
|
|
config['Properties']['ImageId'] = self.conf.minimal_image_ref
|
|
|
|
self.update_instance_group(updt_template,
|
|
num_updates_expected_on_updt=5,
|
|
num_creates_expected_on_updt=0,
|
|
num_deletes_expected_on_updt=0,
|
|
update_replace=True)
|
|
|
|
def test_instance_group_update_replace_huge_min_in_service(self):
|
|
"""
|
|
Test update replace with a huge number of minimum instances in service.
|
|
"""
|
|
updt_template = self.ig_tmpl_with_updt_policy()
|
|
group = updt_template['Resources']['JobServerGroup']
|
|
policy = group['UpdatePolicy']['RollingUpdate']
|
|
policy['MinInstancesInService'] = '20'
|
|
policy['MaxBatchSize'] = '2'
|
|
policy['PauseTime'] = 'PT0S'
|
|
config = updt_template['Resources']['JobServerConfig']
|
|
config['Properties']['ImageId'] = self.conf.minimal_image_ref
|
|
|
|
self.update_instance_group(updt_template,
|
|
num_updates_expected_on_updt=3,
|
|
num_creates_expected_on_updt=2,
|
|
num_deletes_expected_on_updt=2,
|
|
update_replace=True)
|
|
|
|
def test_instance_group_update_no_replace(self):
|
|
"""
|
|
Test simple update only and no replace (i.e. updated instance flavor
|
|
in Launch Configuration) with no conflict in batch size and
|
|
minimum instances in service.
|
|
"""
|
|
updt_template = self.ig_tmpl_with_updt_policy()
|
|
group = updt_template['Resources']['JobServerGroup']
|
|
policy = group['UpdatePolicy']['RollingUpdate']
|
|
policy['MinInstancesInService'] = '1'
|
|
policy['MaxBatchSize'] = '3'
|
|
policy['PauseTime'] = 'PT0S'
|
|
config = updt_template['Resources']['JobServerConfig']
|
|
config['Properties']['InstanceType'] = 'm1.tiny'
|
|
|
|
self.update_instance_group(updt_template,
|
|
num_updates_expected_on_updt=5,
|
|
num_creates_expected_on_updt=0,
|
|
num_deletes_expected_on_updt=0,
|
|
update_replace=False)
|
|
|
|
def test_instance_group_update_no_replace_with_adjusted_capacity(self):
|
|
"""
|
|
Test update only and no replace (i.e. updated instance flavor in
|
|
Launch Configuration) with capacity adjustment due to conflict in
|
|
batch size and minimum instances in service.
|
|
"""
|
|
updt_template = self.ig_tmpl_with_updt_policy()
|
|
group = updt_template['Resources']['JobServerGroup']
|
|
policy = group['UpdatePolicy']['RollingUpdate']
|
|
policy['MinInstancesInService'] = '4'
|
|
policy['MaxBatchSize'] = '4'
|
|
policy['PauseTime'] = 'PT0S'
|
|
config = updt_template['Resources']['JobServerConfig']
|
|
config['Properties']['InstanceType'] = 'm1.tiny'
|
|
|
|
self.update_instance_group(updt_template,
|
|
num_updates_expected_on_updt=2,
|
|
num_creates_expected_on_updt=3,
|
|
num_deletes_expected_on_updt=3,
|
|
update_replace=False)
|