openstack/tempest
Code Issues Proposed changes

Heat autoscaling scenario test

Browse Source 
This test starts with a single server and scales up to
three servers triggered by a script that consumes memory.

Seven minutes after stack creation, memory consumption script
will quit and the scale down alarms will scale back down to
a single server.

Due to the nature of this test, it takes about 10 minutes to
run locally.

The scenario test has been put in package orchestration
for the following reasons:
- this will be the first of many heat scenario tests
- this will allow a tox filter to run this test for the
  slow heat gating job

Change-Id: I53ed12369d12b902108b9b8fa7885df34f6ab51f
This commit is contained in:
Steve Baker
2013-06-24 14:46:47 +12:00
parent 7395517a5f
commit dd7c6cef4e
5 changed files with 364 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
requirements.txt
View File

@@ -13,6 +13,7 @@ python-keystoneclient>=0.2.0
python-novaclient>=2.10.0
python-neutronclient>=2.2.3,<3.0.0
python-cinderclient>=1.0.4
python-heatclient>=0.2.3
testresources
keyring
testrepository

80
tempest/scenario/manager.py
View File

@@ -16,11 +16,13 @@
# License for the specific language governing permissions and limitations
# under the License.
import os
import subprocess
# Default client libs
import cinderclient.client
import glanceclient
import heatclient.client
import keystoneclient.v2_0.client
import netaddr
from neutronclient.common import exceptions as exc
@@ -48,6 +50,7 @@ class OfficialClientManager(tempest.manager.Manager):
NOVACLIENT_VERSION = '2'
CINDERCLIENT_VERSION = '1'
HEATCLIENT_VERSION = '1'
def __init__(self, username, password, tenant_name):
super(OfficialClientManager, self).__init__()
@@ -62,6 +65,10 @@ class OfficialClientManager(tempest.manager.Manager):
self.volume_client = self._get_volume_client(username,
password,
tenant_name)
self.orchestration_client = self._get_orchestration_client(
username,
password,
tenant_name)
def _get_compute_client(self, username, password, tenant_name):
# Novaclient will not execute operations for anyone but the
@@ -98,6 +105,32 @@ class OfficialClientManager(tempest.manager.Manager):
tenant_name,
auth_url)
def _get_orchestration_client(self, username=None, password=None,
tenant_name=None):
if not username:
username = self.config.identity.admin_username
if not password:
password = self.config.identity.admin_password
if not tenant_name:
tenant_name = self.config.identity.tenant_name
self._validate_credentials(username, password, tenant_name)
keystone = self._get_identity_client(username, password, tenant_name)
token = keystone.auth_token
try:
endpoint = keystone.service_catalog.url_for(
service_type='orchestration',
endpoint_type='publicURL')
except keystoneclient.exceptions.EndpointNotFound:
return None
else:
return heatclient.client.Client(self.HEATCLIENT_VERSION,
endpoint,
token=token,
username=username,
password=password)
def _get_identity_client(self, username, password, tenant_name):
# This identity client is not intended to check the security
# of the identity service, so use admin credentials by default.
@@ -153,13 +186,8 @@ class OfficialClientTest(tempest.test.BaseTestCase):
super(OfficialClientTest, cls).setUpClass()
cls.isolated_creds = isolated_creds.IsolatedCreds(
__name__, tempest_client=False)
if cls.config.compute.allow_tenant_isolation:
creds = cls.isolated_creds.get_primary_creds()
username, tenant_name, password = creds
else:
username = cls.config.identity.username
password = cls.config.identity.password
tenant_name = cls.config.identity.tenant_name
username, tenant_name, password = cls.credentials()
cls.manager = OfficialClientManager(username, password, tenant_name)
cls.compute_client = cls.manager.compute_client
@@ -167,9 +195,20 @@ class OfficialClientTest(tempest.test.BaseTestCase):
cls.identity_client = cls.manager.identity_client
cls.network_client = cls.manager.network_client
cls.volume_client = cls.manager.volume_client
cls.orchestration_client = cls.manager.orchestration_client
cls.resource_keys = {}
cls.os_resources = []
@classmethod
def credentials(cls):
if cls.config.compute.allow_tenant_isolation:
return cls.isolated_creds.get_primary_creds()
username = cls.config.identity.username
password = cls.config.identity.password
tenant_name = cls.config.identity.tenant_name
return username, tenant_name, password
@classmethod
def tearDownClass(cls):
# NOTE(jaypipes): Because scenario tests are typically run in a
@@ -498,3 +537,30 @@ class NetworkScenarioTest(OfficialClientTest):
timeout=self.config.compute.ssh_timeout),
'Auth failure in connecting to %s@%s via ssh' %
(username, ip_address))
class OrchestrationScenarioTest(OfficialClientTest):
"""
Base class for orchestration scenario tests
"""
@classmethod
def credentials(cls):
username = cls.config.identity.admin_username
password = cls.config.identity.admin_password
tenant_name = cls.config.identity.tenant_name
return username, tenant_name, password
def _load_template(self, base_file, file_name):
filepath = os.path.join(os.path.dirname(os.path.realpath(base_file)),
file_name)
with open(filepath) as f:
return f.read()
@classmethod
def _stack_rand_name(cls):
return rand_name(cls.__name__ + '-')
def _create_keypair(self):
kp_name = rand_name('keypair-smoke')
return self.compute_client.keypairs.create(kp_name)

0
tempest/scenario/orchestration/__init__.py Normal file
View File

108
tempest/scenario/orchestration/test_autoscaling.py Normal file
View File

@@ -0,0 +1,108 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from tempest.openstack.common import log as logging
from tempest.scenario import manager
from tempest.test import attr
from tempest.test import call_until_true
import time
LOG = logging.getLogger(__name__)
class AutoScalingTest(manager.OrchestrationScenarioTest):
def setUp(self):
super(AutoScalingTest, self).setUp()
if not self.config.orchestration.image_ref:
raise self.skipException("No image available to test")
self.client = self.orchestration_client
def assign_keypair(self):
self.stack_name = self._stack_rand_name()
if self.config.orchestration.keypair_name:
self.keypair_name = self.config.orchestration.keypair_name
else:
self.keypair = self._create_keypair()
self.keypair_name = self.keypair.id
self.set_resource('keypair', self.keypair)
def launch_stack(self):
self.parameters = {
'KeyName': self.keypair_name,
'InstanceType': self.config.orchestration.instance_type,
'ImageId': self.config.orchestration.image_ref,
'StackStart': str(time.time())
}
# create the stack
self.template = self._load_template(__file__, 'test_autoscaling.yaml')
self.client.stacks.create(
stack_name=self.stack_name,
template=self.template,
parameters=self.parameters)
self.stack = self.client.stacks.get(self.stack_name)
self.stack_identifier = '%s/%s' % (self.stack_name, self.stack.id)
# if a keypair was set, do not delete the stack on exit to allow
# for manual post-mortums
if not self.config.orchestration.keypair_name:
self.set_resource('stack', self.stack)
@attr(type='slow')
def test_scale_up_then_down(self):
self.assign_keypair()
self.launch_stack()
sid = self.stack_identifier
timeout = self.config.orchestration.build_timeout
interval = 10
self.assertEqual('CREATE', self.stack.action)
# wait for create to complete.
self.status_timeout(self.client.stacks, sid, 'COMPLETE')
self.stack.get()
self.assertEqual('CREATE_COMPLETE', self.stack.stack_status)
# the resource SmokeServerGroup is implemented as a nested
# stack, so servers can be counted by counting the resources
# inside that nested stack
resource = self.client.resources.get(sid, 'SmokeServerGroup')
nested_stack_id = resource.physical_resource_id
def server_count():
# the number of servers is the number of resources
# in the nexted stack
self.server_count = len(
self.client.resources.list(nested_stack_id))
return self.server_count
def assertScale(from_servers, to_servers):
call_until_true(lambda: server_count() == to_servers,
timeout, interval)
self.assertEqual(to_servers, self.server_count,
'Failed scaling from %d to %d servers' % (
from_servers, to_servers))
# he marched them up to the top of the hill
assertScale(1, 2)
assertScale(2, 3)
# and he marched them down again
assertScale(3, 2)
assertScale(2, 1)

182
tempest/scenario/orchestration/test_autoscaling.yaml Normal file
View File

@@ -0,0 +1,182 @@
HeatTemplateFormatVersion: '2012-12-12'
Description: |
Template which tests autoscaling and load balancing
Parameters:
KeyName:
Type: String
InstanceType:
Type: String
ImageId:
Type: String
StackStart:
Description: Epoch seconds when the stack was launched
Type: Number
ConsumeStartSeconds:
Description: Seconds after invocation when memory should be consumed
Type: Number
Default: '60'
ConsumeStopSeconds:
Description: Seconds after StackStart when memory should be released
Type: Number
Default: '420'
ScaleUpThreshold:
Description: Memory percentage threshold to scale up on
Type: Number
Default: '70'
ScaleDownThreshold:
Description: Memory percentage threshold to scale down on
Type: Number
Default: '60'
ConsumeMemoryLimit:
Description: Memory percentage threshold to consume
Type: Number
Default: '71'
Resources:
SmokeServerGroup:
Type: AWS::AutoScaling::AutoScalingGroup
Properties:
AvailabilityZones: {'Fn::GetAZs': ''}
LaunchConfigurationName: {Ref: LaunchConfig}
MinSize: '1'
MaxSize: '3'
SmokeServerScaleUpPolicy:
Type: AWS::AutoScaling::ScalingPolicy
Properties:
AdjustmentType: ChangeInCapacity
AutoScalingGroupName: {Ref: SmokeServerGroup}
Cooldown: '60'
ScalingAdjustment: '1'
SmokeServerScaleDownPolicy:
Type: AWS::AutoScaling::ScalingPolicy
Properties:
AdjustmentType: ChangeInCapacity
AutoScalingGroupName: {Ref: SmokeServerGroup}
Cooldown: '60'
ScalingAdjustment: '-1'
MEMAlarmHigh:
Type: AWS::CloudWatch::Alarm
Properties:
AlarmDescription: Scale-up if MEM > ScaleUpThreshold% for 10 seconds
MetricName: MemoryUtilization
Namespace: system/linux
Statistic: Average
Period: '10'
EvaluationPeriods: '1'
Threshold: {Ref: ScaleUpThreshold}
AlarmActions: [{Ref: SmokeServerScaleUpPolicy}]
Dimensions:
- Name: AutoScalingGroupName
Value: {Ref: SmokeServerGroup}
ComparisonOperator: GreaterThanThreshold
MEMAlarmLow:
Type: AWS::CloudWatch::Alarm
Properties:
AlarmDescription: Scale-down if MEM < ScaleDownThreshold% for 10 seconds
MetricName: MemoryUtilization
Namespace: system/linux
Statistic: Average
Period: '10'
EvaluationPeriods: '1'
Threshold: {Ref: ScaleDownThreshold}
AlarmActions: [{Ref: SmokeServerScaleDownPolicy}]
Dimensions:
- Name: AutoScalingGroupName
Value: {Ref: SmokeServerGroup}
ComparisonOperator: LessThanThreshold
CfnUser:
Type: AWS::IAM::User
SmokeKeys:
Type: AWS::IAM::AccessKey
Properties:
UserName: {Ref: CfnUser}
SmokeSecurityGroup:
Type: AWS::EC2::SecurityGroup
Properties:
GroupDescription: Standard firewall rules
SecurityGroupIngress:
- {IpProtocol: tcp, FromPort: '22', ToPort: '22', CidrIp: 0.0.0.0/0}
- {IpProtocol: tcp, FromPort: '80', ToPort: '80', CidrIp: 0.0.0.0/0}
LaunchConfig:
Type: AWS::AutoScaling::LaunchConfiguration
Metadata:
AWS::CloudFormation::Init:
config:
files:
/etc/cfn/cfn-credentials:
content:
Fn::Replace:
- $AWSAccessKeyId: {Ref: SmokeKeys}
$AWSSecretKey: {'Fn::GetAtt': [SmokeKeys, SecretAccessKey]}
- |
AWSAccessKeyId=$AWSAccessKeyId
AWSSecretKey=$AWSSecretKey
mode: '000400'
owner: root
group: root
/root/watch_loop:
content:
Fn::Replace:
- _hi_: {Ref: MEMAlarmHigh}
_lo_: {Ref: MEMAlarmLow}
- |
#!/bin/bash
while :
do
/opt/aws/bin/cfn-push-stats --watch _hi_ --mem-util
/opt/aws/bin/cfn-push-stats --watch _lo_ --mem-util
sleep 4
done
mode: '000700'
owner: root
group: root
/root/consume_memory:
content:
Fn::Replace:
- StackStart: {Ref: StackStart}
ConsumeStopSeconds: {Ref: ConsumeStopSeconds}
ConsumeStartSeconds: {Ref: ConsumeStartSeconds}
ConsumeMemoryLimit: {Ref: ConsumeMemoryLimit}
- |
#!/usr/bin/env python
import psutil
import time
import datetime
import sys
a = []
sleep_until_consume = ConsumeStartSeconds
stack_start = StackStart
consume_stop_time = stack_start + ConsumeStopSeconds
memory_limit = ConsumeMemoryLimit
if sleep_until_consume > 0:
sys.stdout.flush()
time.sleep(sleep_until_consume)
while psutil.virtual_memory().percent < memory_limit:
sys.stdout.flush()
a.append(' ' * 10**5)
time.sleep(0.1)
sleep_until_exit = consume_stop_time - time.time()
if sleep_until_exit > 0:
time.sleep(sleep_until_exit)
mode: '000700'
owner: root
group: root
Properties:
ImageId: {Ref: ImageId}
InstanceType: {Ref: InstanceType}
KeyName: {Ref: KeyName}
SecurityGroups: [{Ref: SmokeSecurityGroup}]
UserData:
Fn::Base64:
Fn::Replace:
- ConsumeStopSeconds: {Ref: ConsumeStopSeconds}
ConsumeStartSeconds: {Ref: ConsumeStartSeconds}
ConsumeMemoryLimit: {Ref: ConsumeMemoryLimit}
- |
#!/bin/bash -v
/opt/aws/bin/cfn-init
# report on memory consumption every 4 seconds
/root/watch_loop &
# wait ConsumeStartSeconds then ramp up memory consumption
# until it is over ConsumeMemoryLimit%
# then exits ConsumeStopSeconds seconds after stack launch
/root/consume_memory > /root/consume_memory.log &