Make a template that demonstrates instance monitoring and restarting.
We use cfn-push-stats to send a heartbeat and if we don't get it within the set interval we restart the instance. Other changes to make this work: - clear the waitcondition when it is deleted. - set the resource id to None when deleted, else it won't re-create properly. - don't run watch alarms if the stack is not completed. Change-Id: I5dfc8b372f557cf43379c6c5b7436d1010f83e3c Signed-off-by: Angus Salkeld <asalkeld@redhat.com>
This commit is contained in:
Angus Salkeld
parent
783b111249
commit
3c6e40f2bb
|
|
@ -408,7 +408,12 @@ class EngineManager(manager.Manager):
|
|||
def _periodic_watcher_task(self, context):
|
||||
|
||||
now = timeutils.utcnow()
|
||||
wrs = db_api.watch_rule_get_all(context)
|
||||
try:
|
||||
wrs = db_api.watch_rule_get_all(context)
|
||||
except Exception as ex:
|
||||
logger.warn('periodic_task db error (%s) %s' %
|
||||
('watch rule removed?', str(ex)))
|
||||
return
|
||||
for wr in wrs:
|
||||
# has enough time progressed to run the rule
|
||||
dt_period = datetime.timedelta(seconds=int(wr.rule['Period']))
|
||||
|
|
@ -427,22 +432,25 @@ class EngineManager(manager.Manager):
|
|||
new_state = watcher.get_alarm_state()
|
||||
|
||||
if new_state != wr.state:
|
||||
wr.state = new_state
|
||||
wr.save()
|
||||
logger.warn('WATCH: stack:%s, watch_name:%s %s',
|
||||
wr.stack_name, wr.name, new_state)
|
||||
|
||||
if not action_map[new_state] in wr.rule:
|
||||
logger.info('no action for new state %s',
|
||||
new_state)
|
||||
wr.state = new_state
|
||||
wr.save()
|
||||
else:
|
||||
s = db_api.stack_get_by_name(None, wr.stack_name)
|
||||
if s:
|
||||
if s and s.status in ('CREATE_COMPLETE',
|
||||
'UPDATE_COMPLETE'):
|
||||
user_creds = db_api.user_creds_get(s.user_creds_id)
|
||||
ctxt = ctxtlib.RequestContext.from_dict(dict(user_creds))
|
||||
stack = parser.Stack.load(ctxt, s.id)
|
||||
for a in wr.rule[action_map[new_state]]:
|
||||
greenpool.spawn_n(stack[a].alarm)
|
||||
wr.state = new_state
|
||||
wr.save()
|
||||
|
||||
wr.last_evaluated = now
|
||||
|
||||
|
|
|
|||
|
|
@ -589,9 +589,7 @@ class Stack(object):
|
|||
|
||||
for res in reversed(deps):
|
||||
try:
|
||||
res.delete()
|
||||
re = db_api.resource_get(self.context, res.id)
|
||||
re.delete()
|
||||
res.destroy()
|
||||
except Exception as ex:
|
||||
failed = True
|
||||
logger.error('delete: %s' % str(ex))
|
||||
|
|
|
|||
|
|
@ -342,6 +342,8 @@ class Resource(object):
|
|||
logger.exception('Delete %s from DB' % str(self))
|
||||
return str(ex)
|
||||
|
||||
self.id = None
|
||||
|
||||
def instance_id_set(self, inst):
|
||||
self.instance_id = inst
|
||||
if self.id is not None:
|
||||
|
|
|
|||
|
|
@ -116,6 +116,14 @@ class WaitCondition(resources.Resource):
|
|||
def handle_update(self):
|
||||
return self.UPDATE_REPLACE
|
||||
|
||||
def handle_delete(self):
|
||||
self._get_handle_resource_id()
|
||||
if self.resource_id is None:
|
||||
return
|
||||
|
||||
handle = self.stack[self.resource_id]
|
||||
handle.metadata = {}
|
||||
|
||||
def FnGetAtt(self, key):
|
||||
res = None
|
||||
if key == 'Data':
|
||||
|
|
|
|||
|
|
@ -0,0 +1,232 @@
|
|||
{
|
||||
"AWSTemplateFormatVersion" : "2010-09-09",
|
||||
|
||||
"Description" : "AWS CloudFormation Sample Template WordPress_Multi_Instance: WordPress is web software you can use to create a beautiful website or blog. This template installs two instances: one running a WordPress deployment and the other using a local MySQL database to store the data.",
|
||||
|
||||
"Parameters" : {
|
||||
|
||||
"KeyName" : {
|
||||
"Description" : "Name of an existing EC2 KeyPair to enable SSH access to the instances",
|
||||
"Type" : "String"
|
||||
},
|
||||
|
||||
"InstanceType" : {
|
||||
"Description" : "WebServer EC2 instance type",
|
||||
"Type" : "String",
|
||||
"Default" : "m1.large",
|
||||
"AllowedValues" : [ "t1.micro", "m1.small", "m1.large", "m1.xlarge", "m2.xlarge", "m2.2xlarge", "m2.4xlarge", "c1.medium", "c1.xlarge", "cc1.4xlarge" ],
|
||||
"ConstraintDescription" : "must be a valid EC2 instance type."
|
||||
},
|
||||
|
||||
"DBName": {
|
||||
"Default": "wordpress",
|
||||
"Description" : "The WordPress database name",
|
||||
"Type": "String",
|
||||
"MinLength": "1",
|
||||
"MaxLength": "64",
|
||||
"AllowedPattern" : "[a-zA-Z][a-zA-Z0-9]*",
|
||||
"ConstraintDescription" : "must begin with a letter and contain only alphanumeric characters."
|
||||
},
|
||||
|
||||
"DBUsername": {
|
||||
"Default": "admin",
|
||||
"NoEcho": "true",
|
||||
"Description" : "The WordPress database admin account username",
|
||||
"Type": "String",
|
||||
"MinLength": "1",
|
||||
"MaxLength": "16",
|
||||
"AllowedPattern" : "[a-zA-Z][a-zA-Z0-9]*",
|
||||
"ConstraintDescription" : "must begin with a letter and contain only alphanumeric characters."
|
||||
},
|
||||
|
||||
"DBPassword": {
|
||||
"Default": "admin",
|
||||
"NoEcho": "true",
|
||||
"Description" : "The WordPress database admin account password",
|
||||
"Type": "String",
|
||||
"MinLength": "1",
|
||||
"MaxLength": "41",
|
||||
"AllowedPattern" : "[a-zA-Z0-9]*",
|
||||
"ConstraintDescription" : "must contain only alphanumeric characters."
|
||||
},
|
||||
|
||||
"DBRootPassword": {
|
||||
"Default": "admin",
|
||||
"NoEcho": "true",
|
||||
"Description" : "Root password for MySQL",
|
||||
"Type": "String",
|
||||
"MinLength": "1",
|
||||
"MaxLength": "41",
|
||||
"AllowedPattern" : "[a-zA-Z0-9]*",
|
||||
"ConstraintDescription" : "must contain only alphanumeric characters."
|
||||
},
|
||||
"LinuxDistribution": {
|
||||
"Default": "F16",
|
||||
"Description" : "Distribution of choice",
|
||||
"Type": "String",
|
||||
"AllowedValues" : [ "F16", "F17", "U10", "RHEL-6.1", "RHEL-6.2", "RHEL-6.3" ]
|
||||
},
|
||||
"HupPollInterval": {
|
||||
"Default": "1",
|
||||
"Description" : "Interval for cfn-hup",
|
||||
"Type": "String"
|
||||
}
|
||||
},
|
||||
|
||||
"Mappings" : {
|
||||
"AWSInstanceType2Arch" : {
|
||||
"t1.micro" : { "Arch" : "32" },
|
||||
"m1.small" : { "Arch" : "32" },
|
||||
"m1.large" : { "Arch" : "64" },
|
||||
"m1.xlarge" : { "Arch" : "64" },
|
||||
"m2.xlarge" : { "Arch" : "64" },
|
||||
"m2.2xlarge" : { "Arch" : "64" },
|
||||
"m2.4xlarge" : { "Arch" : "64" },
|
||||
"c1.medium" : { "Arch" : "32" },
|
||||
"c1.xlarge" : { "Arch" : "64" },
|
||||
"cc1.4xlarge" : { "Arch" : "64" }
|
||||
},
|
||||
"DistroArch2AMI": {
|
||||
"F16" : { "32" : "F16-i386-cfntools", "64" : "F16-x86_64-cfntools" },
|
||||
"F17" : { "32" : "F17-i386-cfntools", "64" : "F17-x86_64-cfntools" },
|
||||
"U10" : { "32" : "U10-i386-cfntools", "64" : "U10-x86_64-cfntools" },
|
||||
"RHEL-6.1" : { "32" : "rhel61-i386-cfntools", "64" : "rhel61-x86_64-cfntools" },
|
||||
"RHEL-6.2" : { "32" : "rhel62-i386-cfntools", "64" : "rhel62-x86_64-cfntools" },
|
||||
"RHEL-6.3" : { "32" : "rhel63-i386-cfntools", "64" : "rhel63-x86_64-cfntools" }
|
||||
}
|
||||
},
|
||||
|
||||
"Resources" : {
|
||||
"WebServerRestartPolicy" : {
|
||||
"Type" : "HEAT::HA::Restarter",
|
||||
"Properties" : {
|
||||
"InstanceId" : { "Ref" : "WikiDatabase" }
|
||||
}
|
||||
},
|
||||
"HeartbeatFailureAlarm": {
|
||||
"Type": "AWS::CloudWatch::Alarm",
|
||||
"Properties": {
|
||||
"AlarmDescription": "Restart the WikiDatabase if we miss a heartbeat",
|
||||
"MetricName": "Heartbeat",
|
||||
"Namespace": "system/linux",
|
||||
"Statistic": "SampleCount",
|
||||
"Period": "60",
|
||||
"EvaluationPeriods": "1",
|
||||
"Threshold": "1",
|
||||
"AlarmActions": [ { "Ref": "WebServerRestartPolicy" } ],
|
||||
"ComparisonOperator": "LessThanThreshold"
|
||||
}
|
||||
},
|
||||
"WikiDatabase": {
|
||||
"Type": "AWS::EC2::Instance",
|
||||
"Metadata" : {
|
||||
"AWS::CloudFormation::Init" : {
|
||||
"config" : {
|
||||
"files" : {
|
||||
"/tmp/cfn-hup-crontab.txt" : {
|
||||
"content" : { "Fn::Join" : ["", [
|
||||
"MAIL=\"\"\n",
|
||||
"\n",
|
||||
"* * * * * /opt/aws/bin/cfn-push-stats ",
|
||||
" --watch HeartbeatFailureAlarm --heartbeat\n"
|
||||
]]},
|
||||
"mode" : "000600",
|
||||
"owner" : "root",
|
||||
"group" : "root"
|
||||
},
|
||||
|
||||
"/tmp/setup.mysql" : {
|
||||
"content" : { "Fn::Join" : ["", [
|
||||
"CREATE DATABASE ", { "Ref" : "DBName" }, ";\n",
|
||||
"GRANT ALL PRIVILEGES ON ", { "Ref" : "DBName" },
|
||||
".* TO '", { "Ref" : "DBUsername" }, "'@'localhost'\n",
|
||||
"IDENTIFIED BY '", { "Ref" : "DBPassword" }, "';\n",
|
||||
"FLUSH PRIVILEGES;\n",
|
||||
"EXIT\n"
|
||||
]]},
|
||||
"mode" : "000644",
|
||||
"owner" : "root",
|
||||
"group" : "root"
|
||||
}
|
||||
},
|
||||
"packages" : {
|
||||
"yum" : {
|
||||
"cronie" : [],
|
||||
"mysql" : [],
|
||||
"mysql-server" : [],
|
||||
"httpd" : [],
|
||||
"wordpress" : []
|
||||
}
|
||||
},
|
||||
"services" : {
|
||||
"systemd" : {
|
||||
"mysqld" : { "enabled" : "true", "ensureRunning" : "true" },
|
||||
"httpd" : { "enabled" : "true", "ensureRunning" : "true" },
|
||||
"crond" : { "enabled" : "true", "ensureRunning" : "true" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"Properties": {
|
||||
"ImageId" : { "Fn::FindInMap" : [ "DistroArch2AMI", { "Ref" : "LinuxDistribution" },
|
||||
{ "Fn::FindInMap" : [ "AWSInstanceType2Arch", { "Ref" : "InstanceType" }, "Arch" ] } ] },
|
||||
"InstanceType" : { "Ref" : "InstanceType" },
|
||||
"KeyName" : { "Ref" : "KeyName" },
|
||||
"UserData" : { "Fn::Base64" : { "Fn::Join" : ["", [
|
||||
"#!/bin/bash -v\n",
|
||||
"# Helper function\n",
|
||||
"function error_exit\n",
|
||||
"{\n",
|
||||
" /opt/aws/bin/cfn-signal -e 1 -r \"$1\" '", { "Ref" : "WaitHandle" }, "'\n",
|
||||
" exit 1\n",
|
||||
"}\n",
|
||||
|
||||
"/opt/aws/bin/cfn-init -s ", { "Ref" : "AWS::StackName" },
|
||||
" -r WikiDatabase ",
|
||||
" --region ", { "Ref" : "AWS::Region" },
|
||||
" || error_exit 'Failed to run cfn-init'\n",
|
||||
|
||||
"# Setup MySQL root password and create a user\n",
|
||||
"mysqladmin -u root password '", { "Ref" : "DBRootPassword" },
|
||||
"' || error_exit 'Failed to initialize root password'\n",
|
||||
|
||||
"mysql -u root --password='", { "Ref" : "DBRootPassword" },
|
||||
"' < /tmp/setup.mysql || error_exit 'Failed to create database.'\n",
|
||||
|
||||
"sed --in-place --e s/database_name_here/", { "Ref" : "DBName" },
|
||||
"/ --e s/username_here/", { "Ref" : "DBUsername" },
|
||||
"/ --e s/password_here/", { "Ref" : "DBPassword" },
|
||||
"/ /usr/share/wordpress/wp-config.php\n",
|
||||
|
||||
"# install cfn-hup crontab\n",
|
||||
"crontab /tmp/cfn-hup-crontab.txt\n",
|
||||
|
||||
"# All is well so signal success\n",
|
||||
"/opt/aws/bin/cfn-signal -e 0 -r \"Wiki server setup complete\" '",
|
||||
{ "Ref" : "WaitHandle" }, "'\n"
|
||||
]]}}
|
||||
}
|
||||
},
|
||||
|
||||
"WaitHandle" : {
|
||||
"Type" : "AWS::CloudFormation::WaitConditionHandle"
|
||||
},
|
||||
|
||||
"WaitCondition" : {
|
||||
"Type" : "AWS::CloudFormation::WaitCondition",
|
||||
"DependsOn" : "WikiDatabase",
|
||||
"Properties" : {
|
||||
"Handle" : {"Ref" : "WaitHandle"},
|
||||
"Timeout" : "600"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
"Outputs" : {
|
||||
"WebsiteURL" : {
|
||||
"Value" : { "Fn::Join" : ["", ["http://", { "Fn::GetAtt" : [ "WikiDatabase", "PublicIp" ]}, "/wordpress"]] },
|
||||
"Description" : "URL for Wordpress wiki"
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue