cloudwatch: set HA template to send watch data

The Wordpress HA template is now able to utilize the cfn-push-stats and send
alarm messages to the metadata server.

Change-Id: I52b615d3401dc2665e2b30e4a925d61ed204c827
Signed-off-by: Tomas Sedovic <tomas@sedovic.cz>

MANIFEST.in

@@ -20,6 +20,7 @@ include heat/cfntools/cfn-init
 include heat/cfntools/cfn-hup
 include heat/cfntools/cfn-signal
 include heat/cfntools/cfn-get-metadata
+include heat/cfntools/cfn-push-stats
 include heat/cloudinit/config
 include heat/cloudinit/part-handler.py
 include heat/db/sqlalchemy/migrate_repo/migrate.cfg

heat/cfntools/cfn-hup

@@ -100,12 +100,6 @@ if not mainconfig.unique_resources_get():
     exit(1)
 
 
-metadata_handler = MetadataLoggingHandler(metadata_server_url(),
-                                          mainconfig.stack,
-                                          mainconfig.unique_resources_get()[0])
-logger.addHandler(metadata_handler)
-
-
 for r in mainconfig.unique_resources_get():
     print r
     metadata = Metadata(mainconfig.stack,

heat/cfntools/cfn-push-stats

@@ -17,11 +17,20 @@ Implements cfn-signal CloudFormation functionality
 """
 import argparse
 import logging
-import psutil
 import os
 import random
 import sys
 
+
+log_format = '%(levelname)s [%(asctime)s] %(message)s'
+logging.basicConfig(format=log_format, level=logging.DEBUG)
+logger = logging.getLogger('cfn-push-stats')
+
+try:
+    import psutil
+except ImportError:
+    logger.warn("psutil not available. If you want process and memory "
+            "statistics, you need to install it.")
 #
 #  --aws-credential-file=PATH  Specifies the location of the file with AWS
 #                              credentials.
@@ -84,14 +93,10 @@ parser.add_argument('--disk-units', required=False, default='megabytes',
                     help='Specifies units for disk metrics.')
 parser.add_argument('--disk-path', required=False, default='/',
                     help='Selects the disk by the path on which to report.')
-parser.add_argument('url',
+parser.add_argument('--watch', required=True,
-                    help='the url to post to')
+                    help='the name of the watch to post to.')
 args = parser.parse_args()
 
-log_format = '%(levelname)s [%(asctime)s] %(message)s'
-logging.basicConfig(format=log_format, level=logging.DEBUG)
-logger = logging.getLogger('cfn-push-stats')
-
 data = {'Namespace': 'system/linux'}
 
 # service failure
@@ -150,9 +155,15 @@ if args.disk_space_avail:
 
 logger.info(str(data))
 
+server_url = metadata_server_url()
+if not server_url:
+    logger.error('can not get the metadata_server_url')
+    exit(1)
+url = '%sstats/%s/data/' % (server_url, args.watch)
+
 cmd_str = "curl -X POST -H \'Content-Type:\' --data-binary \'%s\' %s" % \
-           (json.dumps(data), args.url)
+           (json.dumps(data), url)
 
 cmd = CommandRunner(cmd_str)
 cmd.run()
-print cmd.stdout
+logger.info(cmd.stdout)

heat/cfntools/cfn_helper.py

@@ -767,36 +767,6 @@ def metadata_server_url():
         return None
 
 
-class MetadataLoggingHandler(logging.Handler):
-    def __init__(self, metadata_server_url, stack, resource):
-        super(MetadataLoggingHandler, self).__init__(level=logging.WARNING)
-        self.stack = stack
-        self.resource = resource
-
-        if metadata_server_url:
-            self.events_url = metadata_server_url + 'events/'
-        else:
-            logger.info('Metadata server URL not found')
-            self.events_url = None
-
-    def handle(self, record):
-        if not self.events_url:
-            return
-        event = {
-            'message': record.message,
-            'stack': self.stack,
-            'resource': self.resource,
-            'resource_type': 'AWS::EC2::Instance',
-            'reason': 'cfntools notification',
-        }
-        req = Request(self.events_url, json.dumps(event))
-        req.headers['Content-Type'] = 'application/json'
-        try:
-            urlopen(req)
-        except:
-            pass
-
-
 class MetadataServerConnectionError(Exception):
     pass
 

heat/common/exception.py

@@ -19,6 +19,8 @@
 
 import functools
 import urlparse
+import sys
+
 from heat.openstack.common.exception import *
 
 

heat/engine/checkeddict.py

@@ -79,6 +79,10 @@ class CheckedDict(collections.MutableMapping):
                 if num > maxn or num < minn:
                     raise ValueError('%s is out of range' % key)
 
+            elif self.data[key]['Type'] == 'List':
+                if not isinstance(value, list):
+                    raise ValueError('%s Value must be a list' % (key))
+
             elif self.data[key]['Type'] == 'CommaDelimitedList':
                 sp = value.split(',')
                 if not isinstance(sp, list):

heat/engine/cloud_watch.py

@@ -30,6 +30,7 @@ class CloudWatchAlarm(Resource):
                          'AllowedValues': ['GreaterThanOrEqualToThreshold',
                          'GreaterThanThreshold', 'LessThanThreshold',
                          'LessThanOrEqualToThreshold']},
+        'AlarmDescription': {'Type': 'String'},
         'EvaluationPeriods': {'Type': 'String'},
         'MetricName': {'Type': 'String'},
         'Namespace': {'Type': 'String'},
@@ -37,6 +38,7 @@ class CloudWatchAlarm(Resource):
         'Statistic': {'Type': 'String',
                       'AllowedValues': ['SampleCount', 'Average', 'Sum',
                                         'Minimum', 'Maximum']},
+        'AlarmActions': {'Type': 'List'},
         'Threshold': {'Type': 'String'},
         'Units': {'Type': 'String',
                   'AllowedValues': ['Seconds', 'Microseconds', 'Milliseconds',
@@ -90,3 +92,6 @@ class CloudWatchAlarm(Resource):
 
     def FnGetRefId(self):
         return unicode(self.name)
+
+    def strict_dependency(self):
+        return False

heat/engine/escalation_policy.py

@@ -0,0 +1,64 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import eventlet
+import logging
+import json
+import os
+
+from heat.common import exception
+from heat.db import api as db_api
+from heat.engine.resources import Resource
+
+logger = logging.getLogger('heat.engine.escalation_policy')
+
+
+class EscalationPolicy(Resource):
+    properties_schema = {
+            'Instance': {'Type': 'String'},
+            }
+
+    def __init__(self, name, json_snippet, stack):
+        super(EscalationPolicy, self).__init__(name, json_snippet, stack)
+        self.instance_id = ''
+
+    def validate(self):
+        '''
+        Validate the Properties
+        '''
+        return Resource.validate(self)
+
+    def create(self):
+        if self.state != None:
+            return
+        self.state_set(self.CREATE_IN_PROGRESS)
+        Resource.create(self)
+        self.state_set(self.CREATE_COMPLETE)
+
+    def delete(self):
+        if self.state == self.DELETE_IN_PROGRESS or \
+           self.state == self.DELETE_COMPLETE:
+            return
+
+        self.state_set(self.DELETE_IN_PROGRESS)
+
+        Resource.delete(self)
+        self.state_set(self.DELETE_COMPLETE)
+
+    def FnGetRefId(self):
+        return unicode(self.name)
+
+    def strict_dependency(self):
+        return False

heat/engine/parser.py

@@ -21,6 +21,7 @@ from heat.common import exception
 from heat.engine import checkeddict
 from heat.engine import cloud_watch
 from heat.engine import eip
+from heat.engine import escalation_policy
 from heat.engine import instance
 from heat.engine import resources
 from heat.engine import security_group
@@ -39,6 +40,8 @@ logger = logging.getLogger(__file__)
     'AWS::EC2::EIP': eip.ElasticIp,
     'AWS::EC2::EIPAssociation': eip.ElasticIpAssociation,
     'AWS::EC2::SecurityGroup': security_group.SecurityGroup,
+    'AWS::CloudWatch::Alarm': cloud_watch.CloudWatchAlarm,
+    'HEAT::Recovery::EscalationPolicy': escalation_policy.EscalationPolicy,
     'AWS::CloudFormation::WaitConditionHandle':
         wait_condition.WaitConditionHandle,
     'AWS::CloudFormation::WaitCondition': wait_condition.WaitCondition,
@@ -110,7 +113,6 @@ class Stack(object):
         # TODO(sdake) Should return line number of invalid reference
 
         response = None
-
         try:
             order = self.get_create_order()
         except KeyError:
@@ -289,8 +291,9 @@ class Stack(object):
                     pass
                 elif i == 'Ref':
                     #print '%s Refences %s' % (r.name, s[i])
-                    r.depends_on.append(s[i])
+                    if r.strict_dependency():
-                elif i == 'DependsOn' or i == 'Ref':
+                        r.depends_on.append(s[i])
+                elif i == 'DependsOn':
                     #print '%s DependsOn on %s' % (r.name, s[i])
                     r.depends_on.append(s[i])
                 else:

heat/engine/resources.py

@@ -187,7 +187,7 @@ class Resource(object):
         http://docs.amazonwebservices.com/AWSCloudFormation/latest/UserGuide/ \
         intrinsic-function-reference-getatt.html
         '''
-        raise exception.InvalidTemplateAttribute(resource=self.name, key=key)
+        return unicode(self.name)
 
     def FnBase64(self, data):
         '''
@@ -196,6 +196,12 @@ class Resource(object):
         '''
         return base64.b64encode(data)
 
+    def strict_dependency(self):
+        '''
+        If True, this resource must be created before it can be referenced.
+        '''
+        return True
+
 
 class GenericResource(Resource):
     properties_schema = {}

heat/engine/wait_condition.py

@@ -33,6 +33,7 @@ class WaitConditionHandle(Resource):
     then the cfn-signal will use this url to post to and
     WaitCondition will poll it to see if has been written to.
     '''
+    properties_schema = {}
 
     def __init__(self, name, json_snippet, stack):
         super(WaitConditionHandle, self).__init__(name, json_snippet, stack)

heat/jeos/F16-i386-cfntools-jeos.tdl

@@ -23,5 +23,6 @@ EOF
     <file name='/opt/aws/bin/cfn-signal'  type='base64'></file>
     <file name='/opt/aws/bin/cfn_helper.py'  type='base64'></file>
     <file name='/opt/aws/bin/cfn-get-metadata'  type='base64'></file>
+    <file name='/opt/aws/bin/cfn-push-stats'  type='base64'></file>
   </files>
 </template>

heat/jeos/F16-x86_64-cfntools-jeos.tdl

@@ -23,5 +23,6 @@ EOF
     <file name='/opt/aws/bin/cfn-signal'  type='base64'></file>
     <file name='/opt/aws/bin/cfn_helper.py'  type='base64'></file>
     <file name='/opt/aws/bin/cfn-get-metadata'  type='base64'></file>
+    <file name='/opt/aws/bin/cfn-push-stats'  type='base64'></file>
   </files>
 </template>

heat/jeos/F17-i386-cfntools-jeos.tdl

@@ -23,5 +23,6 @@ EOF
     <file name='/opt/aws/bin/cfn-signal'  type='base64'></file>
     <file name='/opt/aws/bin/cfn_helper.py'  type='base64'></file>
     <file name='/opt/aws/bin/cfn-get-metadata'  type='base64'></file>
+    <file name='/opt/aws/bin/cfn-push-stats'  type='base64'></file>
   </files>
 </template>

heat/jeos/F17-x86_64-cfntools-jeos.tdl

@@ -23,6 +23,6 @@ EOF
     <file name='/opt/aws/bin/cfn-signal'  type='base64'></file>
     <file name='/opt/aws/bin/cfn_helper.py'  type='base64'></file>
     <file name='/opt/aws/bin/cfn-get-metadata'  type='base64'></file>
+    <file name='/opt/aws/bin/cfn-push-stats'  type='base64'></file>
   </files>
-
 </template>

heat/jeos/U10-amd64-cfntools-jeos.tdl

@@ -20,5 +20,6 @@ apt-get -y upgrade;apt-get -y install cloud-init;/usr/sbin/useradd -m ec2-user;e
     <file name='/opt/aws/bin/cfn-signal'  type='base64'></file>
     <file name='/opt/aws/bin/cfn_helper.py'  type='base64'></file>
     <file name='/opt/aws/bin/cfn-get-metadata'  type='base64'></file>
+    <file name='/opt/aws/bin/cfn-push-stats'  type='base64'></file>
   </files>
 </template>

heat/utils.py

@@ -140,8 +140,8 @@ def jeos_create(options, arguments, jeos_path, cfntools_path):
     # and injecting them into the TDL at the appropriate place
     if instance_type == 'cfntools':
         tdl_xml = etree.parse(tdl_path)
-        cfn_tools = ['cfn-init', 'cfn-hup', 'cfn-signal', \
+        cfn_tools = ['cfn-init', 'cfn-hup', 'cfn-signal',
-                    'cfn-get-metadata', 'cfn_helper.py']
+                    'cfn-get-metadata', 'cfn_helper.py', 'cfn-push-stats']
         for cfnname in cfn_tools:
             f = open('%s/%s' % (cfntools_path, cfnname), 'r')
             cfscript_e64 = base64.b64encode(f.read())

templates/WordPress_Single_Instance_With_HA.template

@@ -100,19 +100,19 @@
     "WebServerRestartPolicy" : {
       "Type" : "HEAT::Recovery::EscalationPolicy",
       "Properties" : {
-        "Instance" : { "Ref" : "WikiDatabase" },
+        "Instance" : { "Ref" : "WikiDatabase" }
       }
     },
     "HttpFailureAlarm": {
      "Type": "AWS::CloudWatch::Alarm",
      "Properties": {
         "AlarmDescription": "Restart the WikiDatabase if httpd fails > 3 times in 10 minutes",
-        "MetricName": "ProcessRestartCount",
+        "MetricName": "ServiceFailure",
-        "Namespace": "HEAT",
+        "Namespace": "system/linux",
         "Statistic": "Maximum",
         "Period": "300",
-        "EvaluationPeriods": "2",
+        "EvaluationPeriods": "1",
-        "Threshold": "3",
+        "Threshold": "2",
         "AlarmActions": [ { "Ref": "WebServerRestartPolicy" } ],
         "ComparisonOperator": "GreaterThanThreshold"
       }
@@ -123,31 +123,6 @@
         "AWS::CloudFormation::Init" : {
           "config" : {
             "files" : {
-              "/opt/aws/bin/cfn-init" : {
-                "source" : "https://raw.github.com/heat-api/heat/master/heat/cfntools/cfn-init",
-                "mode"   : "000755",
-                "owner"  : "root",
-                "group"  : "root"
-              },
-              "/opt/aws/bin/cfn-hup" : {
-                "source" : "https://raw.github.com/heat-api/heat/master/heat/cfntools/cfn-hup",
-                "mode"   : "000755",
-                "owner"  : "root",
-                "group"  : "root"
-              },
-              "/opt/aws/bin/cfn-signal" : {
-                "source" : "https://raw.github.com/heat-api/heat/master/heat/cfntools/cfn-signal",
-                "mode"   : "000755",
-                "owner"  : "root",
-                "group"  : "root"
-              },
-              "/opt/aws/bin/cfn_helper.py" : {
-                "source" : "https://raw.github.com/heat-api/heat/master/heat/cfntools/cfn_helper.py",
-                "mode"   : "000644",
-                "owner"  : "root",
-                "group"  : "root"
-              },
-
               "/etc/cfn/cfn-credentials" : {
                 "content" : { "Fn::Join" : ["", [
                   "AWSAccessKeyId=GobbleGobble\n",
@@ -174,7 +149,9 @@
               "/etc/cfn/notify-on-httpd-restarted" : {
                 "content" : { "Fn::Join" : ["", [
                 "#!/bin/sh\n",
-                "logger -t cfn-event 'http got restarted'\n"
+                "/opt/aws/bin/cfn-push-stats --watch ",
+                { "Ref" : "HttpFailureAlarm" },
+                " --service-failure\n"
                 ]]},
                 "mode"    : "000700",
                 "owner"   : "root",