# vim: tabstop=4 shiftwidth=4 softtabstop=4 # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import eventlet import json import functools import copy import logging from heat.common import exception from heat.engine import checkeddict from heat.engine import dependencies from heat.engine.resources import Resource from heat.db import api as db_api logger = logging.getLogger('heat.engine.parser') SECTIONS = (VERSION, DESCRIPTION, MAPPINGS, PARAMETERS, RESOURCES, OUTPUTS) = \ ('AWSTemplateFormatVersion', 'Description', 'Mappings', 'Parameters', 'Resources', 'Outputs') (PARAM_STACK_NAME, PARAM_REGION) = ('AWS::StackName', 'AWS::Region') class Parameters(checkeddict.CheckedDict): ''' The parameters of a stack, with type checking, defaults &c. specified by the stack's template. ''' def __init__(self, stack_name, template, user_params={}): ''' Create the parameter container for a stack from the stack name and template, optionally setting the initial set of parameters. ''' checkeddict.CheckedDict.__init__(self, PARAMETERS) self._init_schemata(template[PARAMETERS]) self[PARAM_STACK_NAME] = stack_name self.update(user_params) def _init_schemata(self, schemata): ''' Initialise the parameter schemata with the pseudo-parameters and the list of schemata obtained from the template. ''' self.addschema(PARAM_STACK_NAME, {"Description": "AWS StackName", "Type": "String"}) self.addschema(PARAM_REGION, { "Description": "AWS Regions", "Default": "ap-southeast-1", "Type": "String", "AllowedValues": ["us-east-1", "us-west-1", "us-west-2", "sa-east-1", "eu-west-1", "ap-southeast-1", "ap-northeast-1"], "ConstraintDescription": "must be a valid EC2 instance type.", }) for param, schema in schemata.items(): self.addschema(param, copy.deepcopy(schema)) def user_parameters(self): ''' Return a dictionary of all the parameters passed in by the user ''' return dict((k, v['Value']) for k, v in self.data.iteritems() if 'Value' in v) class Template(object): '''A stack template.''' def __init__(self, template, template_id=None): ''' Initialise the template with a JSON object and a set of Parameters ''' self.id = template_id self.t = template self.maps = self[MAPPINGS] @classmethod def load(cls, context, template_id): '''Retrieve a Template with the given ID from the database''' t = db_api.raw_template_get(context, template_id) return cls(t.template, template_id) def store(self, context=None): '''Store the Template in the database and return its ID''' if self.id is None: rt = {'template': self.t} new_rt = db_api.raw_template_create(context, rt) self.id = new_rt.id return self.id def __getitem__(self, section): '''Get the relevant section in the template''' if section not in SECTIONS: raise KeyError('"%s" is not a valid template section' % section) if section == VERSION: return self.t[section] if section == DESCRIPTION: default = 'No description' else: default = {} return self.t.get(section, default) def resolve_find_in_map(self, s): ''' Resolve constructs of the form { "Fn::FindInMap" : [ "mapping", "key", "value" ] } ''' def handle_find_in_map(args): try: name, key, value = args return self.maps[name][key][value] except (ValueError, TypeError) as ex: raise KeyError(str(ex)) return _resolve(lambda k, v: k == 'Fn::FindInMap', handle_find_in_map, s) @staticmethod def resolve_availability_zones(s): ''' looking for { "Fn::GetAZs" : "str" } ''' def match_get_az(key, value): return (key == 'Fn::GetAZs' and isinstance(value, basestring)) def handle_get_az(ref): return ['nova'] return _resolve(match_get_az, handle_get_az, s) @staticmethod def resolve_param_refs(s, parameters): ''' Resolve constructs of the form { "Ref" : "string" } ''' def match_param_ref(key, value): return (key == 'Ref' and isinstance(value, basestring) and value in parameters) def handle_param_ref(ref): try: return parameters[ref] except (KeyError, ValueError): raise exception.UserParameterMissing(key=ref) return _resolve(match_param_ref, handle_param_ref, s) @staticmethod def resolve_resource_refs(s, resources): ''' Resolve constructs of the form { "Ref" : "resource" } ''' def match_resource_ref(key, value): return key == 'Ref' and value in resources def handle_resource_ref(arg): return resources[arg].FnGetRefId() return _resolve(match_resource_ref, handle_resource_ref, s) @staticmethod def resolve_attributes(s, resources): ''' Resolve constructs of the form { "Fn::GetAtt" : [ "WebServer", "PublicIp" ] } ''' def handle_getatt(args): resource, att = args try: return resources[resource].FnGetAtt(att) except KeyError: raise exception.InvalidTemplateAttribute(resource=resource, key=att) return _resolve(lambda k, v: k == 'Fn::GetAtt', handle_getatt, s) @staticmethod def resolve_joins(s): ''' Resolve constructs of the form { "Fn::Join" : [ "delim", [ "str1", "str2" ] } ''' def handle_join(args): if not isinstance(args, (list, tuple)): raise TypeError('Arguments to "Fn::Join" must be a list') delim, strings = args if not isinstance(strings, (list, tuple)): raise TypeError('Arguments to "Fn::Join" not fully resolved') return delim.join(strings) return _resolve(lambda k, v: k == 'Fn::Join', handle_join, s) @staticmethod def resolve_base64(s): ''' Resolve constructs of the form { "Fn::Base64" : "string" } ''' def handle_base64(string): if not isinstance(string, basestring): raise TypeError('Arguments to "Fn::Base64" not fully resolved') return string return _resolve(lambda k, v: k == 'Fn::Base64', handle_base64, s) class Stack(object): IN_PROGRESS = 'IN_PROGRESS' CREATE_FAILED = 'CREATE_FAILED' CREATE_COMPLETE = 'CREATE_COMPLETE' DELETE_IN_PROGRESS = 'DELETE_IN_PROGRESS' DELETE_FAILED = 'DELETE_FAILED' DELETE_COMPLETE = 'DELETE_COMPLETE' def __init__(self, context, stack_name, template, parameters=None, stack_id=None): ''' Initialise from a context, name, Template object and (optionally) Parameters object. The database ID may also be initialised, if the stack is already in the database. ''' self.id = stack_id self.context = context self.t = template self.name = stack_name if parameters is None: parameters = Parameters(stack_name, template) self.parameters = parameters self.outputs = self.resolve_static_data(self.t[OUTPUTS]) self.resources = dict((name, Resource(name, data, self)) for (name, data) in self.t[RESOURCES].items()) self.dependencies = self._get_dependencies(self.resources.itervalues()) @staticmethod def _get_dependencies(resources): '''Return the dependency graph for a list of resources''' deps = dependencies.Dependencies() for resource in resources: resource.add_dependencies(deps) return deps @classmethod def load(cls, context, stack_id): '''Retrieve a Stack from the database''' s = db_api.stack_get(context, stack_id) template = Template.load(context, s.raw_template_id) params = Parameters(s.name, template, s.parameters) stack = cls(context, s.name, template, params, stack_id) return stack def store(self, owner=None): '''Store the stack in the database and return its ID''' if self.id is None: new_creds = db_api.user_creds_create(self.context.to_dict()) s = {'name': self.name, 'raw_template_id': self.t.store(), 'parameters': self.parameters.user_parameters(), 'owner_id': owner and owner.id, 'user_creds_id': new_creds.id, 'username': self.context.username} new_s = db_api.stack_create(self.context, s) self.id = new_s.id return self.id def __iter__(self): ''' Return an iterator over this template's resources in the order that they should be started. ''' return iter(self.dependencies) def __reversed__(self): ''' Return an iterator over this template's resources in the order that they should be stopped. ''' return reversed(self.dependencies) def __len__(self): '''Return the number of resources''' return len(self.resources) def __getitem__(self, key): '''Get the resource with the specified name.''' return self.resources[key] def __contains__(self, key): '''Determine whether the stack contains the specified resource''' return key in self.resources def keys(self): '''Return a list of resource keys for the stack''' return self.resources.keys() def __str__(self): '''Return a human-readable string representation of the stack''' return 'Stack "%s"' % self.name def stack_id(self): ''' Return a unique ID for this stack suitable for displaying to the user ''' return '/'.join([self.name, str(self.id)]) def validate(self): ''' http://docs.amazonwebservices.com/AWSCloudFormation/latest/\ APIReference/API_ValidateTemplate.html ''' # TODO(sdake) Should return line number of invalid reference for res in self: try: result = res.validate() except Exception as ex: logger.exception('validate') result = str(ex) if result: err_str = 'Malformed Query Response %s' % result response = {'Description': err_str, 'Parameters': []} return response def format_param(p): return {'NoEcho': 'false', 'ParameterKey': p, 'Description': self.parameters.get_attr(p, 'Description'), 'DefaultValue': self.parameters.get_attr(p, 'Default')} response = {'Description': 'Successfully validated', 'Parameters': [format_param(p) for p in self.parameters]} return response def state_set(self, new_status, reason): '''Update the stack state in the database''' if self.id is None: return stack = db_api.stack_get(self.context, self.id) stack.update_and_save({'status': new_status, 'status_reason': reason}) def create(self, timeout_in_minutes=60): ''' Create the stack and all of the resources. Creation will fail if it exceeds the specified timeout. The default is 60 minutes. ''' self.state_set(self.IN_PROGRESS, 'Stack creation started') stack_status = self.CREATE_COMPLETE reason = 'Stack successfully created' res = None with eventlet.Timeout(timeout_in_minutes * 60) as tmo: try: for res in self: if stack_status != self.CREATE_FAILED: result = res.create() if result: stack_status = self.CREATE_FAILED reason = 'Resource %s failed with: %s' % (str(res), result) else: res.state_set(res.CREATE_FAILED, 'Stack creation aborted') except eventlet.Timeout as t: if t is tmo: stack_status = self.CREATE_FAILED reason = 'Timed out waiting for %s' % str(res) else: # not my timeout raise self.state_set(stack_status, reason) def delete(self): ''' Delete all of the resources, and then the stack itself. ''' self.state_set(self.DELETE_IN_PROGRESS, 'Stack deletion started') failures = [] for res in reversed(self): result = res.destroy() if result: failures.append(str(res)) if failures: self.state_set(self.DELETE_FAILED, 'Failed to delete ' + ', '.join(failures)) else: self.state_set(self.DELETE_COMPLETE, 'Deleted successfully') db_api.stack_delete(self.context, self.id) def output(self, key): value = self.outputs[key].get('Value', '') return self.resolve_runtime_data(value) def get_outputs(self): def output_dict(k): return {'Description': self.outputs[k].get('Description', 'No description given'), 'OutputKey': k, 'OutputValue': self.output(k)} return [output_dict(key) for key in self.outputs] def restart_resource(self, resource_name): ''' stop resource_name and all that depend on it start resource_name and all that depend on it ''' deps = self.dependencies[self[resource_name]] failed = False for res in reversed(deps): try: res.delete() re = db_api.resource_get(self.context, res.id) re.delete() except Exception as ex: failed = True logger.error('delete: %s' % str(ex)) for res in deps: if not failed: try: res.create() except Exception as ex: logger.exception('create') failed = True else: res.state_set(res.CREATE_FAILED, 'Resource restart aborted') # TODO(asalkeld) if any of this fails we Should # restart the whole stack def resolve_static_data(self, snippet): return transform(snippet, [functools.partial(self.t.resolve_param_refs, parameters=self.parameters), self.t.resolve_availability_zones, self.t.resolve_find_in_map]) def resolve_runtime_data(self, snippet): return transform(snippet, [functools.partial(self.t.resolve_resource_refs, resources=self.resources), functools.partial(self.t.resolve_attributes, resources=self.resources), self.t.resolve_joins, self.t.resolve_base64]) def transform(data, transformations): ''' Apply each of the transformation functions in the supplied list to the data in turn. ''' for t in transformations: data = t(data) return data def _resolve(match, handle, snippet): ''' Resolve constructs in a snippet of a template. The supplied match function should return True if a particular key-value pair should be substituted, and the handle function should return the correct substitution when passed the argument list as parameters. Returns a copy of the original snippet with the substitutions performed. ''' recurse = lambda s: _resolve(match, handle, s) if isinstance(snippet, dict): if len(snippet) == 1: k, v = snippet.items()[0] if match(k, v): return handle(recurse(v)) return dict((k, recurse(v)) for k, v in snippet.items()) elif isinstance(snippet, list): return [recurse(v) for v in snippet] return snippet