From f3913a9a3e8f0f84f74ba65a1ba7634e231f354a Mon Sep 17 00:00:00 2001 From: Tomas Sedovic Date: Wed, 23 May 2012 17:59:41 +0200 Subject: [PATCH] cloudwatch: set HA template to send watch data The Wordpress HA template is now able to utilize the cfn-push-stats and send alarm messages to the metadata server. Change-Id: I52b615d3401dc2665e2b30e4a925d61ed204c827 Signed-off-by: Tomas Sedovic --- MANIFEST.in | 1 + heat/cfntools/cfn-hup | 6 -- heat/cfntools/cfn-push-stats | 29 ++++++--- heat/cfntools/cfn_helper.py | 30 --------- heat/common/exception.py | 2 + heat/engine/checkeddict.py | 4 ++ heat/engine/cloud_watch.py | 5 ++ heat/engine/escalation_policy.py | 64 +++++++++++++++++++ heat/engine/parser.py | 9 ++- heat/engine/resources.py | 8 ++- heat/engine/wait_condition.py | 1 + heat/jeos/F16-i386-cfntools-jeos.tdl | 1 + heat/jeos/F16-x86_64-cfntools-jeos.tdl | 1 + heat/jeos/F17-i386-cfntools-jeos.tdl | 1 + heat/jeos/F17-x86_64-cfntools-jeos.tdl | 2 +- heat/jeos/U10-amd64-cfntools-jeos.tdl | 1 + heat/utils.py | 4 +- ...WordPress_Single_Instance_With_HA.template | 39 +++-------- 18 files changed, 125 insertions(+), 83 deletions(-) create mode 100644 heat/engine/escalation_policy.py diff --git a/MANIFEST.in b/MANIFEST.in index 3e981ef626..c6886005d0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -20,6 +20,7 @@ include heat/cfntools/cfn-init include heat/cfntools/cfn-hup include heat/cfntools/cfn-signal include heat/cfntools/cfn-get-metadata +include heat/cfntools/cfn-push-stats include heat/cloudinit/config include heat/cloudinit/part-handler.py include heat/db/sqlalchemy/migrate_repo/migrate.cfg diff --git a/heat/cfntools/cfn-hup b/heat/cfntools/cfn-hup index b81a727b1c..9aa2613980 100755 --- a/heat/cfntools/cfn-hup +++ b/heat/cfntools/cfn-hup @@ -100,12 +100,6 @@ if not mainconfig.unique_resources_get(): exit(1) -metadata_handler = MetadataLoggingHandler(metadata_server_url(), - mainconfig.stack, - mainconfig.unique_resources_get()[0]) -logger.addHandler(metadata_handler) - - for r in mainconfig.unique_resources_get(): print r metadata = Metadata(mainconfig.stack, diff --git a/heat/cfntools/cfn-push-stats b/heat/cfntools/cfn-push-stats index 996adbe718..148f41ac27 100755 --- a/heat/cfntools/cfn-push-stats +++ b/heat/cfntools/cfn-push-stats @@ -17,11 +17,20 @@ Implements cfn-signal CloudFormation functionality """ import argparse import logging -import psutil import os import random import sys + +log_format = '%(levelname)s [%(asctime)s] %(message)s' +logging.basicConfig(format=log_format, level=logging.DEBUG) +logger = logging.getLogger('cfn-push-stats') + +try: + import psutil +except ImportError: + logger.warn("psutil not available. If you want process and memory " + "statistics, you need to install it.") # # --aws-credential-file=PATH Specifies the location of the file with AWS # credentials. @@ -84,14 +93,10 @@ parser.add_argument('--disk-units', required=False, default='megabytes', help='Specifies units for disk metrics.') parser.add_argument('--disk-path', required=False, default='/', help='Selects the disk by the path on which to report.') -parser.add_argument('url', - help='the url to post to') +parser.add_argument('--watch', required=True, + help='the name of the watch to post to.') args = parser.parse_args() -log_format = '%(levelname)s [%(asctime)s] %(message)s' -logging.basicConfig(format=log_format, level=logging.DEBUG) -logger = logging.getLogger('cfn-push-stats') - data = {'Namespace': 'system/linux'} # service failure @@ -150,9 +155,15 @@ if args.disk_space_avail: logger.info(str(data)) +server_url = metadata_server_url() +if not server_url: + logger.error('can not get the metadata_server_url') + exit(1) +url = '%sstats/%s/data/' % (server_url, args.watch) + cmd_str = "curl -X POST -H \'Content-Type:\' --data-binary \'%s\' %s" % \ - (json.dumps(data), args.url) + (json.dumps(data), url) cmd = CommandRunner(cmd_str) cmd.run() -print cmd.stdout +logger.info(cmd.stdout) diff --git a/heat/cfntools/cfn_helper.py b/heat/cfntools/cfn_helper.py index e6cd0ad110..4429ee1c69 100644 --- a/heat/cfntools/cfn_helper.py +++ b/heat/cfntools/cfn_helper.py @@ -767,36 +767,6 @@ def metadata_server_url(): return None -class MetadataLoggingHandler(logging.Handler): - def __init__(self, metadata_server_url, stack, resource): - super(MetadataLoggingHandler, self).__init__(level=logging.WARNING) - self.stack = stack - self.resource = resource - - if metadata_server_url: - self.events_url = metadata_server_url + 'events/' - else: - logger.info('Metadata server URL not found') - self.events_url = None - - def handle(self, record): - if not self.events_url: - return - event = { - 'message': record.message, - 'stack': self.stack, - 'resource': self.resource, - 'resource_type': 'AWS::EC2::Instance', - 'reason': 'cfntools notification', - } - req = Request(self.events_url, json.dumps(event)) - req.headers['Content-Type'] = 'application/json' - try: - urlopen(req) - except: - pass - - class MetadataServerConnectionError(Exception): pass diff --git a/heat/common/exception.py b/heat/common/exception.py index dd2eed5ee9..be718ecd10 100644 --- a/heat/common/exception.py +++ b/heat/common/exception.py @@ -19,6 +19,8 @@ import functools import urlparse +import sys + from heat.openstack.common.exception import * diff --git a/heat/engine/checkeddict.py b/heat/engine/checkeddict.py index 0a0cdb0488..ce9a41cfe9 100644 --- a/heat/engine/checkeddict.py +++ b/heat/engine/checkeddict.py @@ -79,6 +79,10 @@ class CheckedDict(collections.MutableMapping): if num > maxn or num < minn: raise ValueError('%s is out of range' % key) + elif self.data[key]['Type'] == 'List': + if not isinstance(value, list): + raise ValueError('%s Value must be a list' % (key)) + elif self.data[key]['Type'] == 'CommaDelimitedList': sp = value.split(',') if not isinstance(sp, list): diff --git a/heat/engine/cloud_watch.py b/heat/engine/cloud_watch.py index 6f5d38c6a6..8e1d4a1953 100644 --- a/heat/engine/cloud_watch.py +++ b/heat/engine/cloud_watch.py @@ -30,6 +30,7 @@ class CloudWatchAlarm(Resource): 'AllowedValues': ['GreaterThanOrEqualToThreshold', 'GreaterThanThreshold', 'LessThanThreshold', 'LessThanOrEqualToThreshold']}, + 'AlarmDescription': {'Type': 'String'}, 'EvaluationPeriods': {'Type': 'String'}, 'MetricName': {'Type': 'String'}, 'Namespace': {'Type': 'String'}, @@ -37,6 +38,7 @@ class CloudWatchAlarm(Resource): 'Statistic': {'Type': 'String', 'AllowedValues': ['SampleCount', 'Average', 'Sum', 'Minimum', 'Maximum']}, + 'AlarmActions': {'Type': 'List'}, 'Threshold': {'Type': 'String'}, 'Units': {'Type': 'String', 'AllowedValues': ['Seconds', 'Microseconds', 'Milliseconds', @@ -90,3 +92,6 @@ class CloudWatchAlarm(Resource): def FnGetRefId(self): return unicode(self.name) + + def strict_dependency(self): + return False diff --git a/heat/engine/escalation_policy.py b/heat/engine/escalation_policy.py new file mode 100644 index 0000000000..33b1d76f19 --- /dev/null +++ b/heat/engine/escalation_policy.py @@ -0,0 +1,64 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import eventlet +import logging +import json +import os + +from heat.common import exception +from heat.db import api as db_api +from heat.engine.resources import Resource + +logger = logging.getLogger('heat.engine.escalation_policy') + + +class EscalationPolicy(Resource): + properties_schema = { + 'Instance': {'Type': 'String'}, + } + + def __init__(self, name, json_snippet, stack): + super(EscalationPolicy, self).__init__(name, json_snippet, stack) + self.instance_id = '' + + def validate(self): + ''' + Validate the Properties + ''' + return Resource.validate(self) + + def create(self): + if self.state != None: + return + self.state_set(self.CREATE_IN_PROGRESS) + Resource.create(self) + self.state_set(self.CREATE_COMPLETE) + + def delete(self): + if self.state == self.DELETE_IN_PROGRESS or \ + self.state == self.DELETE_COMPLETE: + return + + self.state_set(self.DELETE_IN_PROGRESS) + + Resource.delete(self) + self.state_set(self.DELETE_COMPLETE) + + def FnGetRefId(self): + return unicode(self.name) + + def strict_dependency(self): + return False diff --git a/heat/engine/parser.py b/heat/engine/parser.py index 56fc524cdb..45d916c412 100644 --- a/heat/engine/parser.py +++ b/heat/engine/parser.py @@ -21,6 +21,7 @@ from heat.common import exception from heat.engine import checkeddict from heat.engine import cloud_watch from heat.engine import eip +from heat.engine import escalation_policy from heat.engine import instance from heat.engine import resources from heat.engine import security_group @@ -39,6 +40,8 @@ logger = logging.getLogger(__file__) 'AWS::EC2::EIP': eip.ElasticIp, 'AWS::EC2::EIPAssociation': eip.ElasticIpAssociation, 'AWS::EC2::SecurityGroup': security_group.SecurityGroup, + 'AWS::CloudWatch::Alarm': cloud_watch.CloudWatchAlarm, + 'HEAT::Recovery::EscalationPolicy': escalation_policy.EscalationPolicy, 'AWS::CloudFormation::WaitConditionHandle': wait_condition.WaitConditionHandle, 'AWS::CloudFormation::WaitCondition': wait_condition.WaitCondition, @@ -110,7 +113,6 @@ class Stack(object): # TODO(sdake) Should return line number of invalid reference response = None - try: order = self.get_create_order() except KeyError: @@ -289,8 +291,9 @@ class Stack(object): pass elif i == 'Ref': #print '%s Refences %s' % (r.name, s[i]) - r.depends_on.append(s[i]) - elif i == 'DependsOn' or i == 'Ref': + if r.strict_dependency(): + r.depends_on.append(s[i]) + elif i == 'DependsOn': #print '%s DependsOn on %s' % (r.name, s[i]) r.depends_on.append(s[i]) else: diff --git a/heat/engine/resources.py b/heat/engine/resources.py index 6a36e40dab..02a0adce6e 100644 --- a/heat/engine/resources.py +++ b/heat/engine/resources.py @@ -187,7 +187,7 @@ class Resource(object): http://docs.amazonwebservices.com/AWSCloudFormation/latest/UserGuide/ \ intrinsic-function-reference-getatt.html ''' - raise exception.InvalidTemplateAttribute(resource=self.name, key=key) + return unicode(self.name) def FnBase64(self, data): ''' @@ -196,6 +196,12 @@ class Resource(object): ''' return base64.b64encode(data) + def strict_dependency(self): + ''' + If True, this resource must be created before it can be referenced. + ''' + return True + class GenericResource(Resource): properties_schema = {} diff --git a/heat/engine/wait_condition.py b/heat/engine/wait_condition.py index 2371bc6144..b847a0262f 100644 --- a/heat/engine/wait_condition.py +++ b/heat/engine/wait_condition.py @@ -33,6 +33,7 @@ class WaitConditionHandle(Resource): then the cfn-signal will use this url to post to and WaitCondition will poll it to see if has been written to. ''' + properties_schema = {} def __init__(self, name, json_snippet, stack): super(WaitConditionHandle, self).__init__(name, json_snippet, stack) diff --git a/heat/jeos/F16-i386-cfntools-jeos.tdl b/heat/jeos/F16-i386-cfntools-jeos.tdl index d7f7b431ec..0d11021830 100644 --- a/heat/jeos/F16-i386-cfntools-jeos.tdl +++ b/heat/jeos/F16-i386-cfntools-jeos.tdl @@ -23,5 +23,6 @@ EOF + diff --git a/heat/jeos/F16-x86_64-cfntools-jeos.tdl b/heat/jeos/F16-x86_64-cfntools-jeos.tdl index 7b618a7591..dc135adfb8 100644 --- a/heat/jeos/F16-x86_64-cfntools-jeos.tdl +++ b/heat/jeos/F16-x86_64-cfntools-jeos.tdl @@ -23,5 +23,6 @@ EOF + diff --git a/heat/jeos/F17-i386-cfntools-jeos.tdl b/heat/jeos/F17-i386-cfntools-jeos.tdl index 9749e42aed..aff94e9f17 100644 --- a/heat/jeos/F17-i386-cfntools-jeos.tdl +++ b/heat/jeos/F17-i386-cfntools-jeos.tdl @@ -23,5 +23,6 @@ EOF + diff --git a/heat/jeos/F17-x86_64-cfntools-jeos.tdl b/heat/jeos/F17-x86_64-cfntools-jeos.tdl index 17e8869d3e..583bf9de48 100644 --- a/heat/jeos/F17-x86_64-cfntools-jeos.tdl +++ b/heat/jeos/F17-x86_64-cfntools-jeos.tdl @@ -23,6 +23,6 @@ EOF + - diff --git a/heat/jeos/U10-amd64-cfntools-jeos.tdl b/heat/jeos/U10-amd64-cfntools-jeos.tdl index 5ea4db184b..316e39de26 100644 --- a/heat/jeos/U10-amd64-cfntools-jeos.tdl +++ b/heat/jeos/U10-amd64-cfntools-jeos.tdl @@ -20,5 +20,6 @@ apt-get -y upgrade;apt-get -y install cloud-init;/usr/sbin/useradd -m ec2-user;e + diff --git a/heat/utils.py b/heat/utils.py index d008a10d44..61d528b071 100644 --- a/heat/utils.py +++ b/heat/utils.py @@ -140,8 +140,8 @@ def jeos_create(options, arguments, jeos_path, cfntools_path): # and injecting them into the TDL at the appropriate place if instance_type == 'cfntools': tdl_xml = etree.parse(tdl_path) - cfn_tools = ['cfn-init', 'cfn-hup', 'cfn-signal', \ - 'cfn-get-metadata', 'cfn_helper.py'] + cfn_tools = ['cfn-init', 'cfn-hup', 'cfn-signal', + 'cfn-get-metadata', 'cfn_helper.py', 'cfn-push-stats'] for cfnname in cfn_tools: f = open('%s/%s' % (cfntools_path, cfnname), 'r') cfscript_e64 = base64.b64encode(f.read()) diff --git a/templates/WordPress_Single_Instance_With_HA.template b/templates/WordPress_Single_Instance_With_HA.template index 34286dffd7..653668dbb2 100644 --- a/templates/WordPress_Single_Instance_With_HA.template +++ b/templates/WordPress_Single_Instance_With_HA.template @@ -100,19 +100,19 @@ "WebServerRestartPolicy" : { "Type" : "HEAT::Recovery::EscalationPolicy", "Properties" : { - "Instance" : { "Ref" : "WikiDatabase" }, + "Instance" : { "Ref" : "WikiDatabase" } } }, "HttpFailureAlarm": { "Type": "AWS::CloudWatch::Alarm", "Properties": { "AlarmDescription": "Restart the WikiDatabase if httpd fails > 3 times in 10 minutes", - "MetricName": "ProcessRestartCount", - "Namespace": "HEAT", + "MetricName": "ServiceFailure", + "Namespace": "system/linux", "Statistic": "Maximum", "Period": "300", - "EvaluationPeriods": "2", - "Threshold": "3", + "EvaluationPeriods": "1", + "Threshold": "2", "AlarmActions": [ { "Ref": "WebServerRestartPolicy" } ], "ComparisonOperator": "GreaterThanThreshold" } @@ -123,31 +123,6 @@ "AWS::CloudFormation::Init" : { "config" : { "files" : { - "/opt/aws/bin/cfn-init" : { - "source" : "https://raw.github.com/heat-api/heat/master/heat/cfntools/cfn-init", - "mode" : "000755", - "owner" : "root", - "group" : "root" - }, - "/opt/aws/bin/cfn-hup" : { - "source" : "https://raw.github.com/heat-api/heat/master/heat/cfntools/cfn-hup", - "mode" : "000755", - "owner" : "root", - "group" : "root" - }, - "/opt/aws/bin/cfn-signal" : { - "source" : "https://raw.github.com/heat-api/heat/master/heat/cfntools/cfn-signal", - "mode" : "000755", - "owner" : "root", - "group" : "root" - }, - "/opt/aws/bin/cfn_helper.py" : { - "source" : "https://raw.github.com/heat-api/heat/master/heat/cfntools/cfn_helper.py", - "mode" : "000644", - "owner" : "root", - "group" : "root" - }, - "/etc/cfn/cfn-credentials" : { "content" : { "Fn::Join" : ["", [ "AWSAccessKeyId=GobbleGobble\n", @@ -174,7 +149,9 @@ "/etc/cfn/notify-on-httpd-restarted" : { "content" : { "Fn::Join" : ["", [ "#!/bin/sh\n", - "logger -t cfn-event 'http got restarted'\n" + "/opt/aws/bin/cfn-push-stats --watch ", + { "Ref" : "HttpFailureAlarm" }, + " --service-failure\n" ]]}, "mode" : "000700", "owner" : "root",