From 5b4a403f51a4099bd8651c8e92101c4f3e5f7e6f Mon Sep 17 00:00:00 2001 From: Zane Bitter Date: Tue, 3 Jul 2012 13:02:03 +0200 Subject: [PATCH] Make Stacks the owners of their own DB representation Change the way that Stack objects are initialised, so that they know how to load and store themselves in the database (they were already updating their own data, but the actual creation was being done externally). This consolidates a lot of existing Stack database creation code (in the manager, nested stacks and unit tests) into one place. Also, split the template and parameter handling out into separate classes, and pass these through the constructor for easier testing. Change-Id: I65bec175191713d0a4a6aa1d3d5442a1b64042f8 Signed-off-by: Zane Bitter --- heat/engine/manager.py | 157 ++++----- heat/engine/parser.py | 429 ++++++++++++++++--------- heat/engine/stack.py | 39 +-- heat/tests/functional/test_bin_heat.py | 12 +- heat/tests/test_parser.py | 228 ++++++++++++- heat/tests/test_resources.py | 13 +- heat/tests/test_stacks.py | 179 ++++++----- heat/tests/test_validate.py | 8 +- heat/tests/test_waitcondition.py | 30 +- 9 files changed, 707 insertions(+), 388 deletions(-) diff --git a/heat/engine/manager.py b/heat/engine/manager.py index 117d22d24b..cce0747792 100644 --- a/heat/engine/manager.py +++ b/heat/engine/manager.py @@ -105,15 +105,12 @@ class EngineManager(manager.Manager): if stacks is None: return res for s in stacks: - ps = parser.Stack(context, s.name, - s.raw_template.template, - s.id, s.parameters) + stack = parser.Stack.load(context, s.id) mem = {} mem['StackId'] = "/".join([s.name, str(s.id)]) mem['StackName'] = s.name mem['CreationTime'] = heat_utils.strtime(s.created_at) - mem['TemplateDescription'] = ps.t.get('Description', - 'No description') + mem['TemplateDescription'] = stack.t[parser.DESCRIPTION] mem['StackStatus'] = s.status res['stacks'].append(mem) @@ -144,24 +141,21 @@ class EngineManager(manager.Manager): logging.debug("Processing show_stack for %s" % stack) s = db_api.stack_get_by_name(context, stack) if s: - ps = parser.Stack(context, s.name, - s.raw_template.template, - s.id, s.parameters) + stack = parser.Stack.load(context, s.id) mem = {} mem['StackId'] = "/".join([s.name, str(s.id)]) mem['StackName'] = s.name mem['CreationTime'] = heat_utils.strtime(s.created_at) mem['LastUpdatedTimestamp'] = heat_utils.strtime(s.updated_at) mem['NotificationARNs'] = 'TODO' - mem['Parameters'] = ps.t['Parameters'] - mem['Description'] = ps.t.get('Description', - 'No description') + mem['Parameters'] = stack.t[parser.PARAMETERS] + mem['Description'] = stack.t[parser.DESCRIPTION] mem['StackStatus'] = s.status mem['StackStatusReason'] = s.status_reason # only show the outputs on a completely created stack - if s.status == ps.CREATE_COMPLETE: - mem['Outputs'] = ps.get_outputs() + if s.status == stack.CREATE_COMPLETE: + mem['Outputs'] = stack.get_outputs() res['stacks'].append(mem) @@ -185,40 +179,19 @@ class EngineManager(manager.Manager): if db_api.stack_get_by_name(None, stack_name): return {'Error': 'Stack already exists with that name.'} - user_params = _extract_user_params(params) - # We don't want to reset the stack template, so we are making - # an instance just for validation. - template_copy = deepcopy(template) - stack_validator = parser.Stack(context, stack_name, - template_copy, 0, - user_params) - response = stack_validator.validate() - stack_validator = None - template_copy = None - if 'Malformed Query Response' in \ - response['ValidateTemplateResult']['Description']: + tmpl = parser.Template(template) + user_params = parser.Parameters(stack_name, tmpl, + _extract_user_params(params)) + stack = parser.Stack(context, stack_name, tmpl, user_params) + + response = stack.validate() + if response['Description'] != 'Successfully validated': return response - stack = parser.Stack(context, stack_name, template, 0, user_params) - rt = {} - rt['template'] = template - rt['StackName'] = stack_name - new_rt = db_api.raw_template_create(None, rt) - - new_creds = db_api.user_creds_create(context.to_dict()) - - s = {} - s['name'] = stack_name - s['raw_template_id'] = new_rt.id - s['user_creds_id'] = new_creds.id - s['username'] = context.username - s['parameters'] = user_params - new_s = db_api.stack_create(context, s) - stack.id = new_s.id - + stack_id = stack.store() greenpool.spawn_n(stack.create, **_extract_args(params)) - return {'StackId': "/".join([new_s.name, str(new_s.id)])} + return {'StackId': "/".join([stack.name, str(stack.id)])} def validate_template(self, context, template, params): """ @@ -237,21 +210,22 @@ class EngineManager(manager.Manager): msg = _("No Template provided.") return webob.exc.HTTPBadRequest(explanation=msg) + stack_name = 'validate' try: - s = parser.Stack(context, 'validate', template, 0, - _extract_user_params(params)) + tmpl = parser.Template(template) + user_params = parser.Parameters(stack_name, tmpl, + _extract_user_params(params)) + s = parser.Stack(context, stack_name, tmpl, user_params) except KeyError as ex: res = ('A Fn::FindInMap operation referenced ' 'a non-existent map [%s]' % str(ex)) - response = {'ValidateTemplateResult': { - 'Description': 'Malformed Query Response [%s]' % (res), - 'Parameters': []}} - return response + result = {'Description': 'Malformed Query Response [%s]' % (res), + 'Parameters': []} + else: + result = s.validate() - res = s.validate() - - return res + return {'ValidateTemplateResult': result} def get_template(self, context, stack_name, params): """ @@ -282,10 +256,8 @@ class EngineManager(manager.Manager): logger.info('deleting stack %s' % stack_name) - ps = parser.Stack(context, st.name, - st.raw_template.template, - st.id, st.parameters) - greenpool.spawn_n(ps.delete) + stack = parser.Stack.load(context, st.id) + greenpool.spawn_n(stack.delete) return None # Helper for list_events. It's here so we can use it in tests. @@ -356,38 +328,43 @@ class EngineManager(manager.Manager): def describe_stack_resource(self, context, stack_name, resource_name): auth.authenticate(context) - stack = db_api.stack_get_by_name(context, stack_name) - if not stack: + s = db_api.stack_get_by_name(context, stack_name) + if not s: raise AttributeError('Unknown stack name') - resource = db_api.resource_get_by_name_and_stack(context, - resource_name, - stack.id) - if not resource: + + stack = parser.Stack.load(context, s.id) + if resource_name not in stack: raise AttributeError('Unknown resource name') - return format_resource_attributes(stack, resource) + + resource = stack[resource_name] + if resource.id is None: + raise AttributeError('Resource not created') + + return format_stack_resource(stack[resource_name]) def describe_stack_resources(self, context, stack_name, physical_resource_id, logical_resource_id): auth.authenticate(context) - if stack_name: - stack = db_api.stack_get_by_name(context, stack_name) + if stack_name is not None: + s = db_api.stack_get_by_name(context, stack_name) else: - resource = db_api.resource_get_by_physical_resource_id(context, + rs = db_api.resource_get_by_physical_resource_id(context, physical_resource_id) - if not resource: + if not rs: msg = "The specified PhysicalResourceId doesn't exist" raise AttributeError(msg) - stack = resource.stack + s = rs.stack - if not stack: + if not s: raise AttributeError("The specified stack doesn't exist") + stack = parser.Stack.load(context, s.id) resources = [] - for r in stack.resources: - if logical_resource_id and r.name != logical_resource_id: + for resource in stack: + if logical_resource_id and resource.name != logical_resource_id: continue - formatted = format_resource_attributes(stack, r) + formatted = format_stack_resource(resource) # this API call uses Timestamp instead of LastUpdatedTimestamp formatted['Timestamp'] = formatted['LastUpdatedTimestamp'] del formatted['LastUpdatedTimestamp'] @@ -398,16 +375,18 @@ class EngineManager(manager.Manager): def list_stack_resources(self, context, stack_name): auth.authenticate(context) - stack = db_api.stack_get_by_name(context, stack_name) - if not stack: + s = db_api.stack_get_by_name(context, stack_name) + if not s: raise AttributeError('Unknown stack name') + stack = parser.Stack.load(context, s.id) + resources = [] response_keys = ('ResourceStatus', 'LogicalResourceId', 'LastUpdatedTimestamp', 'PhysicalResourceId', 'ResourceType') - for r in stack.resources: - formatted = format_resource_attributes(stack, r) + for resource in stack: + formatted = format_stack_resource(resource) for key in formatted.keys(): if not key in response_keys: del formatted[key] @@ -499,11 +478,9 @@ class EngineManager(manager.Manager): if s: user_creds = db_api.user_creds_get(s.user_creds_id) ctxt = ctxtlib.RequestContext.from_dict(dict(user_creds)) - ps = parser.Stack(ctxt, s.name, - s.raw_template.template, - s.id, s.parameters) + stack = parser.Stack.load(ctxt, s.id) for a in wr.rule[action_map[new_state]]: - greenpool.spawn_n(ps[a].alarm) + greenpool.spawn_n(stack[a].alarm) wr.last_evaluated = now @@ -534,22 +511,20 @@ class EngineManager(manager.Manager): return [None, wd.data] -def format_resource_attributes(stack, resource): +def format_stack_resource(resource): """ Return a representation of the given resource that mathes the API output expectations. """ - template = resource.parsed_template.template - template_resources = template.get('Resources', {}) - resource_type = template_resources.get(resource.name, {}).get('Type', '') - last_updated_time = resource.updated_at or resource.created_at + rs = db_api.resource_get(resource.stack.context, resource.id) + last_updated_time = rs.updated_at or rs.created_at return { - 'StackId': stack.id, - 'StackName': stack.name, + 'StackId': resource.stack.id, + 'StackName': resource.stack.name, 'LogicalResourceId': resource.name, - 'PhysicalResourceId': resource.nova_instance or '', - 'ResourceType': resource_type, + 'PhysicalResourceId': resource.instance_id or '', + 'ResourceType': resource.t['Type'], 'LastUpdatedTimestamp': heat_utils.strtime(last_updated_time), - 'ResourceStatus': resource.state, - 'ResourceStatusReason': resource.state_description, + 'ResourceStatus': rs.state, + 'ResourceStatusReason': rs.state_description, } diff --git a/heat/engine/parser.py b/heat/engine/parser.py index 355b68c6f6..ec770a8c48 100644 --- a/heat/engine/parser.py +++ b/heat/engine/parser.py @@ -15,7 +15,8 @@ import eventlet import json -import itertools +import functools +import copy import logging from heat.common import exception @@ -24,8 +25,206 @@ from heat.engine import dependencies from heat.engine.resources import Resource from heat.db import api as db_api + logger = logging.getLogger('heat.engine.parser') +SECTIONS = (VERSION, DESCRIPTION, MAPPINGS, + PARAMETERS, RESOURCES, OUTPUTS) = \ + ('AWSTemplateFormatVersion', 'Description', 'Mappings', + 'Parameters', 'Resources', 'Outputs') + +(PARAM_STACK_NAME, PARAM_REGION) = ('AWS::StackName', 'AWS::Region') + + +class Parameters(checkeddict.CheckedDict): + ''' + The parameters of a stack, with type checking, defaults &c. specified by + the stack's template. + ''' + + def __init__(self, stack_name, template, user_params={}): + ''' + Create the parameter container for a stack from the stack name and + template, optionally setting the initial set of parameters. + ''' + checkeddict.CheckedDict.__init__(self, PARAMETERS) + self._init_schemata(template[PARAMETERS]) + + self[PARAM_STACK_NAME] = stack_name + self.update(user_params) + + def _init_schemata(self, schemata): + ''' + Initialise the parameter schemata with the pseudo-parameters and the + list of schemata obtained from the template. + ''' + self.addschema(PARAM_STACK_NAME, {"Description": "AWS StackName", + "Type": "String"}) + self.addschema(PARAM_REGION, { + "Description": "AWS Regions", + "Default": "ap-southeast-1", + "Type": "String", + "AllowedValues": ["us-east-1", "us-west-1", "us-west-2", + "sa-east-1", "eu-west-1", "ap-southeast-1", + "ap-northeast-1"], + "ConstraintDescription": "must be a valid EC2 instance type.", + }) + + for param, schema in schemata.items(): + self.addschema(param, copy.deepcopy(schema)) + + def user_parameters(self): + ''' + Return a dictionary of all the parameters passed in by the user + ''' + return dict((k, v['Value']) for k, v in self.data.iteritems() + if 'Value' in v) + + +class Template(object): + '''A stack template.''' + + def __init__(self, template, template_id=None): + ''' + Initialise the template with a JSON object and a set of Parameters + ''' + self.id = template_id + self.t = template + self.maps = self[MAPPINGS] + + @classmethod + def load(cls, context, template_id): + '''Retrieve a Template with the given ID from the database''' + t = db_api.raw_template_get(context, template_id) + return cls(t.template, template_id) + + def store(self, context=None): + '''Store the Template in the database and return its ID''' + if self.id is None: + rt = {'template': self.t} + new_rt = db_api.raw_template_create(context, rt) + self.id = new_rt.id + return self.id + + def __getitem__(self, section): + '''Get the relevant section in the template''' + if section not in SECTIONS: + raise KeyError('"%s" is not a valid template section' % section) + if section == VERSION: + return self.t[section] + + if section == DESCRIPTION: + default = 'No description' + else: + default = {} + + return self.t.get(section, default) + + def resolve_find_in_map(self, s): + ''' + Resolve constructs of the form { "Fn::FindInMap" : [ "mapping", + "key", + "value" ] } + ''' + def handle_find_in_map(args): + try: + name, key, value = args + return self.maps[name][key][value] + except (ValueError, TypeError) as ex: + raise KeyError(str(ex)) + + return _resolve(lambda k, v: k == 'Fn::FindInMap', + handle_find_in_map, s) + + @staticmethod + def resolve_availability_zones(s): + ''' + looking for { "Fn::GetAZs" : "str" } + ''' + def match_get_az(key, value): + return (key == 'Fn::GetAZs' and + isinstance(value, basestring)) + + def handle_get_az(ref): + return ['nova'] + + return _resolve(match_get_az, handle_get_az, s) + + @staticmethod + def resolve_param_refs(s, parameters): + ''' + Resolve constructs of the form { "Ref" : "string" } + ''' + def match_param_ref(key, value): + return (key == 'Ref' and + isinstance(value, basestring) and + value in parameters) + + def handle_param_ref(ref): + try: + return parameters[ref] + except (KeyError, ValueError): + raise exception.UserParameterMissing(key=ref) + + return _resolve(match_param_ref, handle_param_ref, s) + + @staticmethod + def resolve_resource_refs(s, resources): + ''' + Resolve constructs of the form { "Ref" : "resource" } + ''' + def match_resource_ref(key, value): + return key == 'Ref' and value in resources + + def handle_resource_ref(arg): + return resources[arg].FnGetRefId() + + return _resolve(match_resource_ref, handle_resource_ref, s) + + @staticmethod + def resolve_attributes(s, resources): + ''' + Resolve constructs of the form { "Fn::GetAtt" : [ "WebServer", + "PublicIp" ] } + ''' + def handle_getatt(args): + resource, att = args + try: + return resources[resource].FnGetAtt(att) + except KeyError: + raise exception.InvalidTemplateAttribute(resource=resource, + key=att) + + return _resolve(lambda k, v: k == 'Fn::GetAtt', handle_getatt, s) + + @staticmethod + def resolve_joins(s): + ''' + Resolve constructs of the form { "Fn::Join" : [ "delim", [ "str1", + "str2" ] } + ''' + def handle_join(args): + if not isinstance(args, (list, tuple)): + raise TypeError('Arguments to "Fn::Join" must be a list') + delim, strings = args + if not isinstance(strings, (list, tuple)): + raise TypeError('Arguments to "Fn::Join" not fully resolved') + return delim.join(strings) + + return _resolve(lambda k, v: k == 'Fn::Join', handle_join, s) + + @staticmethod + def resolve_base64(s): + ''' + Resolve constructs of the form { "Fn::Base64" : "string" } + ''' + def handle_base64(string): + if not isinstance(string, basestring): + raise TypeError('Arguments to "Fn::Base64" not fully resolved') + return string + + return _resolve(lambda k, v: k == 'Fn::Base64', handle_base64, s) + class Stack(object): IN_PROGRESS = 'IN_PROGRESS' @@ -35,46 +234,65 @@ class Stack(object): DELETE_FAILED = 'DELETE_FAILED' DELETE_COMPLETE = 'DELETE_COMPLETE' - def __init__(self, context, stack_name, template, stack_id=0, parms=None): + def __init__(self, context, stack_name, template, parameters=None, + stack_id=None): + ''' + Initialise from a context, name, Template object and (optionally) + Parameters object. The database ID may also be initialised, if the + stack is already in the database. + ''' self.id = stack_id self.context = context self.t = template - self.maps = self.t.get('Mappings', {}) - self.res = {} - self.doc = None self.name = stack_name - # Default Parameters - self.parms = checkeddict.CheckedDict('Parameters') - self.parms.addschema('AWS::StackName', {"Description": "AWS StackName", - "Type": "String"}) - self.parms['AWS::StackName'] = stack_name - self.parms.addschema('AWS::Region', {"Description": "AWS Regions", - "Default": "ap-southeast-1", - "Type": "String", - "AllowedValues": ["us-east-1", "us-west-1", "us-west-2", - "sa-east-1", "eu-west-1", "ap-southeast-1", - "ap-northeast-1"], - "ConstraintDescription": "must be a valid EC2 instance type."}) + if parameters is None: + parameters = Parameters(stack_name, template) + self.parameters = parameters - # template Parameters - ps = self.t.get('Parameters', {}) - for p in ps: - self.parms.addschema(p, ps[p]) - - # user Parameters - if parms is not None: - self.parms.update(parms) - - self.outputs = self.resolve_static_data(self.t.get('Outputs', {})) + self.outputs = self.resolve_static_data(self.t[OUTPUTS]) self.resources = dict((name, Resource(name, data, self)) - for (name, data) in self.t['Resources'].items()) + for (name, data) in self.t[RESOURCES].items()) - self.dependencies = dependencies.Dependencies() - for resource in self.resources.values(): - resource.add_dependencies(self.dependencies) + self.dependencies = self._get_dependencies(self.resources.itervalues()) + + @staticmethod + def _get_dependencies(resources): + '''Return the dependency graph for a list of resources''' + deps = dependencies.Dependencies() + for resource in resources: + resource.add_dependencies(deps) + + return deps + + @classmethod + def load(cls, context, stack_id): + '''Retrieve a Stack from the database''' + s = db_api.stack_get(context, stack_id) + + template = Template.load(context, s.raw_template_id) + params = Parameters(s.name, template, s.parameters) + stack = cls(context, s.name, template, params, stack_id) + + return stack + + def store(self, owner=None): + '''Store the stack in the database and return its ID''' + if self.id is None: + new_creds = db_api.user_creds_create(self.context.to_dict()) + + s = {'name': self.name, + 'raw_template_id': self.t.store(), + 'parameters': self.parameters.user_parameters(), + 'owner_id': owner and owner.id, + 'user_creds_id': new_creds.id, + 'username': self.context.username} + new_s = db_api.stack_create(self.context, s) + self.id = new_s.id + + return self.id def __iter__(self): ''' @@ -103,9 +321,11 @@ class Stack(object): return key in self.resources def keys(self): + '''Return a list of resource keys for the stack''' return self.resources.keys() def __str__(self): + '''Return a human-readable string representation of the stack''' return 'Stack "%s"' % self.name def validate(self): @@ -115,8 +335,6 @@ class Stack(object): ''' # TODO(sdake) Should return line number of invalid reference - response = None - for res in self: try: result = res.validate() @@ -126,35 +344,27 @@ class Stack(object): if result: err_str = 'Malformed Query Response %s' % result - response = {'ValidateTemplateResult': { - 'Description': err_str, - 'Parameters': []}} + response = {'Description': err_str, + 'Parameters': []} return response - if response is None: - response = {'ValidateTemplateResult': { - 'Description': 'Successfully validated', - 'Parameters': []}} - for p in self.parms: - jp = {'member': {}} - res = jp['member'] - res['NoEcho'] = 'false' - res['ParameterKey'] = p - res['Description'] = self.parms.get_attr(p, 'Description') - res['DefaultValue'] = self.parms.get_attr(p, 'Default') - response['ValidateTemplateResult']['Parameters'].append(res) + def format_param(p): + return {'NoEcho': 'false', + 'ParameterKey': p, + 'Description': self.parameters.get_attr(p, 'Description'), + 'DefaultValue': self.parameters.get_attr(p, 'Default')} + + response = {'Description': 'Successfully validated', + 'Parameters': [format_param(p) for p in self.parameters]} + return response - def state_set(self, new_status, reason='change in resource state'): - if self.id != 0: - stack = db_api.stack_get(self.context, self.id) - else: - stack = db_api.stack_get_by_name(self.context, self.name) - - if stack is None: + def state_set(self, new_status, reason): + '''Update the stack state in the database''' + if self.id is None: return - self.id = stack.id + stack = db_api.stack_get(self.context, self.id) stack.update_and_save({'status': new_status, 'status_reason': reason}) @@ -199,7 +409,7 @@ class Stack(object): ''' Delete all of the resources, and then the stack itself. ''' - self.state_set(self.DELETE_IN_PROGRESS) + self.state_set(self.DELETE_IN_PROGRESS, 'Stack deletion started') failures = [] @@ -253,104 +463,25 @@ class Stack(object): logger.exception('create') failed = True else: - res.state_set(res.CREATE_FAILED) + res.state_set(res.CREATE_FAILED, 'Resource restart aborted') # TODO(asalkeld) if any of this fails we Should # restart the whole stack - def parameter_get(self, key): - if not key in self.parms: - raise exception.UserParameterMissing(key=key) - try: - return self.parms[key] - except ValueError: - raise exception.UserParameterMissing(key=key) - - def _resolve_static_refs(self, s): - ''' - looking for { "Ref" : "str" } - ''' - def match(key, value): - return (key == 'Ref' and - isinstance(value, basestring) and - value in self.parms) - - def handle(ref): - return self.parameter_get(ref) - - return _resolve(match, handle, s) - - def _resolve_availability_zones(self, s): - ''' - looking for { "Fn::GetAZs" : "str" } - ''' - def match(key, value): - return (key == 'Fn::GetAZs' and - isinstance(value, basestring)) - - def handle(ref): - return ['nova'] - - return _resolve(match, handle, s) - - def _resolve_find_in_map(self, s): - def handle(args): - try: - name, key, value = args - return self.maps[name][key][value] - except (ValueError, TypeError) as ex: - raise KeyError(str(ex)) - - return _resolve(lambda k, v: k == 'Fn::FindInMap', handle, s) - - def _resolve_attributes(self, s): - ''' - looking for something like: - { "Fn::GetAtt" : [ "DBInstance", "Endpoint.Address" ] } - ''' - def match_ref(key, value): - return key == 'Ref' and value in self - - def handle_ref(arg): - return self[arg].FnGetRefId() - - def handle_getatt(args): - resource, att = args - try: - return self[resource].FnGetAtt(att) - except KeyError: - raise exception.InvalidTemplateAttribute(resource=resource, - key=att) - - return _resolve(lambda k, v: k == 'Fn::GetAtt', handle_getatt, - _resolve(match_ref, handle_ref, s)) - - @staticmethod - def _resolve_joins(s): - ''' - looking for { "Fn::Join" : [] } - ''' - def handle(args): - delim, strings = args - return delim.join(strings) - - return _resolve(lambda k, v: k == 'Fn::Join', handle, s) - - @staticmethod - def _resolve_base64(s): - ''' - looking for { "Fn::Base64" : "" } - ''' - return _resolve(lambda k, v: k == 'Fn::Base64', lambda d: d, s) - def resolve_static_data(self, snippet): - return transform(snippet, [self._resolve_static_refs, - self._resolve_availability_zones, - self._resolve_find_in_map]) + return transform(snippet, + [functools.partial(self.t.resolve_param_refs, + parameters=self.parameters), + self.t.resolve_availability_zones, + self.t.resolve_find_in_map]) def resolve_runtime_data(self, snippet): - return transform(snippet, [self._resolve_attributes, - self._resolve_joins, - self._resolve_base64]) + return transform(snippet, + [functools.partial(self.t.resolve_resource_refs, + resources=self.resources), + functools.partial(self.t.resolve_attributes, + resources=self.resources), + self.t.resolve_joins, + self.t.resolve_base64]) def transform(data, transformations): diff --git a/heat/engine/stack.py b/heat/engine/stack.py index 9bacefa662..a42f7898ea 100644 --- a/heat/engine/stack.py +++ b/heat/engine/stack.py @@ -45,45 +45,30 @@ class Stack(Resource): return p def nested(self): - if self._nested is None: - if self.instance_id is None: - return None + if self._nested is None and self.instance_id is not None: + self._nested = parser.Stack.load(self.stack.context, + self.instance_id) - st = db_api.stack_get(self.stack.context, self.instance_id) - if not st: + if self._nested is None: raise exception.NotFound('Nested stack not found in DB') - n = parser.Stack(self.stack.context, st.name, - st.raw_template.template, - self.instance_id, self._params()) - self._nested = n - return self._nested def create_with_template(self, child_template): ''' Handle the creation of the nested stack from a given JSON template. ''' + template = parser.Template(child_template) + params = parser.Parameters(self.name, template, self._params()) + self._nested = parser.Stack(self.stack.context, self.name, - child_template, - parms=self._params()) - - rt = {'template': child_template, 'stack_name': self.name} - new_rt = db_api.raw_template_create(None, rt) - - parent_stack = db_api.stack_get(self.stack.context, self.stack.id) - - s = {'name': self.name, - 'owner_id': self.stack.id, - 'raw_template_id': new_rt.id, - 'user_creds_id': parent_stack.user_creds_id, - 'username': self.stack.context.username} - new_s = db_api.stack_create(None, s) - self._nested.id = new_s.id + template, + params) + nested_id = self._nested.store(self.stack) + self.instance_id_set(nested_id) self._nested.create() - self.instance_id_set(self._nested.id) def handle_create(self): response = urllib2.urlopen(self.properties[PROP_TEMPLATE_URL]) @@ -110,7 +95,7 @@ class Stack(Resource): if stack is None: # This seems like a hack, to get past validation return '' - if op not in self.nested().outputs: + if op not in stack.outputs: raise exception.InvalidTemplateAttribute(resource=self.name, key=key) diff --git a/heat/tests/functional/test_bin_heat.py b/heat/tests/functional/test_bin_heat.py index f7591605ee..e852eb3087 100644 --- a/heat/tests/functional/test_bin_heat.py +++ b/heat/tests/functional/test_bin_heat.py @@ -206,13 +206,13 @@ class TestBinHeat(): t = json.loads(f.read()) f.close() - params = {} - params['KeyStoneCreds'] = None - t['Parameters']['KeyName']['Value'] = keyname - t['Parameters']['DBUsername']['Value'] = dbusername - t['Parameters']['DBPassword']['Value'] = creds['password'] + template = parser.Template(t) + params = parser.Parameters('test', t, + {'KeyName': keyname, + 'DBUsername': dbusername, + 'DBPassword': creds['password']}) - stack = parser.Stack('test', t, 0, params) + stack = parser.Stack(None, 'test', template, params) parsed_t = stack.resolve_static_refs(t) remote_file = sftp.open('/var/lib/cloud/instance/scripts/startup') remote_file_list = remote_file.read().split('\n') diff --git a/heat/tests/test_parser.py b/heat/tests/test_parser.py index 449e8f46cf..8cc517c781 100644 --- a/heat/tests/test_parser.py +++ b/heat/tests/test_parser.py @@ -1,8 +1,13 @@ import nose import unittest from nose.plugins.attrib import attr +import mox -from heat.engine.parser import _resolve as resolve +import json +from heat.common import exception +from heat.engine import parser +from heat.engine import checkeddict +from heat.engine.resources import Resource def join(raw): @@ -10,7 +15,7 @@ def join(raw): delim, strs = args return delim.join(strs) - return resolve(lambda k, v: k == 'Fn::Join', handle_join, raw) + return parser._resolve(lambda k, v: k == 'Fn::Join', handle_join, raw) @attr(tag=['unit', 'parser']) @@ -76,6 +81,225 @@ class ParserTest(unittest.TestCase): self.assertEqual(join(raw), 'foo bar\nbaz') +mapping_template = json.loads('''{ + "Mappings" : { + "ValidMapping" : { + "TestKey" : { "TestValue" : "wibble" } + }, + "InvalidMapping" : { + "ValueList" : [ "foo", "bar" ], + "ValueString" : "baz" + }, + "MapList": [ "foo", { "bar" : "baz" } ], + "MapString": "foobar" + } +}''') + + +@attr(tag=['unit', 'parser', 'template']) +@attr(speed='fast') +class TemplateTest(unittest.TestCase): + def setUp(self): + self.m = mox.Mox() + + def tearDown(self): + self.m.UnsetStubs() + + def test_defaults(self): + empty = parser.Template({}) + try: + empty[parser.VERSION] + except KeyError: + pass + else: + self.fail('Expected KeyError for version not present') + self.assertEqual(empty[parser.DESCRIPTION], 'No description') + self.assertEqual(empty[parser.MAPPINGS], {}) + self.assertEqual(empty[parser.PARAMETERS], {}) + self.assertEqual(empty[parser.RESOURCES], {}) + self.assertEqual(empty[parser.OUTPUTS], {}) + + def test_invalid_section(self): + tmpl = parser.Template({'Foo': ['Bar']}) + try: + tmpl['Foo'] + except KeyError: + pass + else: + self.fail('Expected KeyError for invalid template key') + + def test_find_in_map(self): + tmpl = parser.Template(mapping_template) + find = {'Fn::FindInMap': ["ValidMapping", "TestKey", "TestValue"]} + self.assertEqual(tmpl.resolve_find_in_map(find), "wibble") + + def test_find_in_invalid_map(self): + tmpl = parser.Template(mapping_template) + finds = ({'Fn::FindInMap': ["InvalidMapping", "ValueList", "foo"]}, + {'Fn::FindInMap': ["InvalidMapping", "ValueString", "baz"]}, + {'Fn::FindInMap': ["MapList", "foo", "bar"]}, + {'Fn::FindInMap': ["MapString", "foo", "bar"]}) + + for find in finds: + self.assertRaises(KeyError, tmpl.resolve_find_in_map, find) + + def test_bad_find_in_map(self): + tmpl = parser.Template(mapping_template) + finds = ({'Fn::FindInMap': "String"}, + {'Fn::FindInMap': {"Dict": "String"}}, + {'Fn::FindInMap': ["ShortList", "foo"]}, + {'Fn::FindInMap': ["ReallyShortList"]}) + + for find in finds: + self.assertRaises(KeyError, tmpl.resolve_find_in_map, find) + + def test_param_refs(self): + params = {'foo': 'bar', 'blarg': 'wibble'} + p_snippet = {"Ref": "foo"} + self.assertEqual(parser.Template.resolve_param_refs(p_snippet, params), + "bar") + + def test_param_refs_resource(self): + params = {'foo': 'bar', 'blarg': 'wibble'} + r_snippet = {"Ref": "baz"} + self.assertEqual(parser.Template.resolve_param_refs(r_snippet, params), + r_snippet) + + def test_param_ref_missing(self): + params = checkeddict.CheckedDict("test") + params.addschema('foo', {"Required": True}) + snippet = {"Ref": "foo"} + self.assertRaises(exception.UserParameterMissing, + parser.Template.resolve_param_refs, + snippet, params) + + def test_resource_refs(self): + resources = {'foo': self.m.CreateMock(Resource), + 'blarg': self.m.CreateMock(Resource)} + resources['foo'].FnGetRefId().AndReturn('bar') + self.m.ReplayAll() + + r_snippet = {"Ref": "foo"} + self.assertEqual(parser.Template.resolve_resource_refs(r_snippet, + resources), + "bar") + self.m.VerifyAll() + + def test_resource_refs_param(self): + resources = {'foo': 'bar', 'blarg': 'wibble'} + p_snippet = {"Ref": "baz"} + self.assertEqual(parser.Template.resolve_resource_refs(p_snippet, + resources), + p_snippet) + + def test_join(self): + join = {"Fn::Join": [" ", ["foo", "bar"]]} + self.assertEqual(parser.Template.resolve_joins(join), "foo bar") + + def test_join_string(self): + join = {"Fn::Join": [" ", "foo"]} + self.assertRaises(TypeError, parser.Template.resolve_joins, + join) + + def test_join_dict(self): + join = {"Fn::Join": [" ", {"foo": "bar"}]} + self.assertRaises(TypeError, parser.Template.resolve_joins, + join) + + def test_join_wrong_num_args(self): + join0 = {"Fn::Join": []} + self.assertRaises(ValueError, parser.Template.resolve_joins, + join0) + join1 = {"Fn::Join": [" "]} + self.assertRaises(ValueError, parser.Template.resolve_joins, + join1) + join3 = {"Fn::Join": [" ", {"foo": "bar"}, ""]} + self.assertRaises(ValueError, parser.Template.resolve_joins, + join3) + + def test_join_string_nodelim(self): + join1 = {"Fn::Join": "o"} + self.assertRaises(TypeError, parser.Template.resolve_joins, + join1) + join2 = {"Fn::Join": "oh"} + self.assertRaises(TypeError, parser.Template.resolve_joins, + join2) + join3 = {"Fn::Join": "ohh"} + self.assertRaises(TypeError, parser.Template.resolve_joins, + join3) + + def test_join_dict_nodelim(self): + join1 = {"Fn::Join": {"foo": "bar"}} + self.assertRaises(TypeError, parser.Template.resolve_joins, + join1) + join2 = {"Fn::Join": {"foo": "bar", "blarg": "wibble"}} + self.assertRaises(TypeError, parser.Template.resolve_joins, + join2) + join3 = {"Fn::Join": {"foo": "bar", "blarg": "wibble", "baz": "quux"}} + self.assertRaises(TypeError, parser.Template.resolve_joins, + join3) + + def test_base64(self): + snippet = {"Fn::Base64": "foobar"} + # For now, the Base64 function just returns the original text, and + # does not convert to base64 (see issue #133) + self.assertEqual(parser.Template.resolve_base64(snippet), "foobar") + + def test_base64_list(self): + list_snippet = {"Fn::Base64": ["foobar"]} + self.assertRaises(TypeError, parser.Template.resolve_base64, + list_snippet) + + def test_base64_dict(self): + dict_snippet = {"Fn::Base64": {"foo": "bar"}} + self.assertRaises(TypeError, parser.Template.resolve_base64, + dict_snippet) + + +params_schema = json.loads('''{ + "Parameters" : { + "User" : { "Type": "String" }, + "Defaulted" : { + "Type": "String", + "Default": "foobar" + } + } +}''') + + +@attr(tag=['unit', 'parser', 'parameters']) +@attr(speed='fast') +class ParametersTest(unittest.TestCase): + def test_pseudo_params(self): + params = parser.Parameters('test_stack', {"Parameters": {}}) + + self.assertEqual(params['AWS::StackName'], 'test_stack') + self.assertTrue('AWS::Region' in params) + + def test_user_param(self): + params = parser.Parameters('test', params_schema, {'User': 'wibble'}) + user_params = params.user_parameters() + self.assertEqual(user_params['User'], 'wibble') + + def test_user_param_default(self): + params = parser.Parameters('test', params_schema) + user_params = params.user_parameters() + self.assertTrue('Defaulted' not in user_params) + + def test_user_param_nonexist(self): + params = parser.Parameters('test', params_schema) + user_params = params.user_parameters() + self.assertTrue('User' not in user_params) + + def test_schema_invariance(self): + params1 = parser.Parameters('test', params_schema) + params1['Defaulted'] = "wibble" + self.assertEqual(params1['Defaulted'], 'wibble') + + params2 = parser.Parameters('test', params_schema) + self.assertEqual(params2['Defaulted'], 'foobar') + + # allows testing of the test directly, shown below if __name__ == '__main__': sys.argv.append(__file__) diff --git a/heat/tests/test_resources.py b/heat/tests/test_resources.py index 0e924ed45a..2b0c2cebbb 100644 --- a/heat/tests/test_resources.py +++ b/heat/tests/test_resources.py @@ -34,19 +34,16 @@ class instancesTest(unittest.TestCase): t = json.loads(f.read()) f.close() - parameters = {} t['Parameters']['KeyName']['Value'] = 'test' - stack = parser.Stack(None, 'test_stack', t, 0) + stack = parser.Stack(None, 'test_stack', parser.Template(t), + stack_id=-1) self.m.StubOutWithMock(db_api, 'resource_get_by_name_and_stack') db_api.resource_get_by_name_and_stack(None, 'test_resource_name', stack).AndReturn(None) self.m.StubOutWithMock(instances.Instance, 'nova') - instances.Instance.nova().AndReturn(self.fc) - instances.Instance.nova().AndReturn(self.fc) - instances.Instance.nova().AndReturn(self.fc) - instances.Instance.nova().AndReturn(self.fc) + instances.Instance.nova().MultipleTimes().AndReturn(self.fc) self.m.ReplayAll() @@ -80,9 +77,9 @@ class instancesTest(unittest.TestCase): t = json.loads(f.read()) f.close() - parameters = {} t['Parameters']['KeyName']['Value'] = 'test' - stack = parser.Stack(None, 'test_stack', t, 0) + stack = parser.Stack(None, 'test_stack', parser.Template(t), + stack_id=-1) self.m.StubOutWithMock(db_api, 'resource_get_by_name_and_stack') db_api.resource_get_by_name_and_stack(None, 'test_resource_name', diff --git a/heat/tests/test_stacks.py b/heat/tests/test_stacks.py index ab758e3770..c234303d4c 100644 --- a/heat/tests/test_stacks.py +++ b/heat/tests/test_stacks.py @@ -24,27 +24,37 @@ class stacksTest(unittest.TestCase): def setUp(self): self.m = mox.Mox() self.fc = fakes.FakeClient() - self.path = os.path.dirname(os.path.realpath(__file__)).\ - replace('heat/tests', 'templates') + path = os.path.dirname(os.path.realpath(__file__)) + self.path = path.replace(os.path.join('heat', 'tests'), 'templates') def tearDown(self): self.m.UnsetStubs() print "stackTest teardown complete" + def create_context(self, user='stacks_test_user'): + ctx = context.get_admin_context() + self.m.StubOutWithMock(ctx, 'username') + ctx.username = user + self.m.StubOutWithMock(auth, 'authenticate') + return ctx + # We use this in a number of tests so it's factored out here. - def start_wordpress_stack(self, stack_name): - f = open("%s/WordPress_Single_Instance_gold.template" % self.path) - t = json.loads(f.read()) - f.close() - params = {} - parameters = {} - t['Parameters']['KeyName']['Value'] = 'test' - stack = parser.Stack(None, stack_name, t, 0, params) + def get_wordpress_stack(self, stack_name, ctx=None): + tmpl_path = os.path.join(self.path, + 'WordPress_Single_Instance_gold.template') + with open(tmpl_path) as f: + t = json.load(f) + + template = parser.Template(t) + parameters = parser.Parameters(stack_name, template, + {'KeyName': 'test'}) + + stack = parser.Stack(ctx or self.create_context(), + stack_name, template, parameters) + self.m.StubOutWithMock(instances.Instance, 'nova') - instances.Instance.nova().AndReturn(self.fc) - instances.Instance.nova().AndReturn(self.fc) - instances.Instance.nova().AndReturn(self.fc) - instances.Instance.nova().AndReturn(self.fc) + instances.Instance.nova().MultipleTimes().AndReturn(self.fc) + instance = stack.resources['WebServer'] instance.itype_oflavor['m1.large'] = 'm1.large' instance.calculate_properties() @@ -55,10 +65,11 @@ class stacksTest(unittest.TestCase): name='WebServer', security_groups=None, userdata=server_userdata).\ AndReturn(self.fc.servers.list()[-1]) + return stack def test_wordpress_single_instance_stack_create(self): - stack = self.start_wordpress_stack('test_stack') + stack = self.get_wordpress_stack('test_stack') self.m.ReplayAll() stack.create() @@ -67,48 +78,28 @@ class stacksTest(unittest.TestCase): self.assertNotEqual(stack.resources['WebServer'].ipaddress, '0.0.0.0') def test_wordpress_single_instance_stack_delete(self): - stack = self.start_wordpress_stack('test_stack') + ctx = self.create_context() + stack = self.get_wordpress_stack('test_stack', ctx) self.m.ReplayAll() - rt = {} - rt['template'] = stack.t - rt['StackName'] = stack.name - new_rt = db_api.raw_template_create(None, rt) - ct = {'username': 'fred', - 'password': 'mentions_fruit'} - new_creds = db_api.user_creds_create(ct) - s = {} - s['name'] = stack.name - s['raw_template_id'] = new_rt.id - s['user_creds_id'] = new_creds.id - s['username'] = ct['username'] - new_s = db_api.stack_create(None, s) - stack.id = new_s.id + stack_id = stack.store() stack.create() + + db_s = db_api.stack_get(ctx, stack_id) + self.assertNotEqual(db_s, None) + self.assertNotEqual(stack.resources['WebServer'], None) self.assertTrue(stack.resources['WebServer'].instance_id > 0) stack.delete() self.assertEqual(stack.resources['WebServer'].state, 'DELETE_COMPLETE') - self.assertEqual(new_s.status, 'DELETE_COMPLETE') + self.assertEqual(db_api.stack_get(ctx, stack_id), None) + self.assertEqual(db_s.status, 'DELETE_COMPLETE') def test_stack_event_list(self): - stack = self.start_wordpress_stack('test_event_list_stack') + stack = self.get_wordpress_stack('test_event_list_stack') self.m.ReplayAll() - rt = {} - rt['template'] = stack.t - rt['StackName'] = stack.name - new_rt = db_api.raw_template_create(None, rt) - ct = {'username': 'fred', - 'password': 'mentions_fruit'} - new_creds = db_api.user_creds_create(ct) - s = {} - s['name'] = stack.name - s['raw_template_id'] = new_rt.id - s['user_creds_id'] = new_creds.id - s['username'] = ct['username'] - new_s = db_api.stack_create(None, s) - stack.id = new_s.id + stack.store() stack.create() self.assertNotEqual(stack.resources['WebServer'], None) @@ -135,47 +126,83 @@ class stacksTest(unittest.TestCase): 'm1.large') def test_stack_list(self): - stack = self.start_wordpress_stack('test_stack_list') - rt = {} - rt['template'] = stack.t - rt['StackName'] = stack.name - new_rt = db_api.raw_template_create(None, rt) - ct = {'username': 'fred', - 'password': 'mentions_fruit'} - new_creds = db_api.user_creds_create(ct) - - ctx = context.get_admin_context() - self.m.StubOutWithMock(ctx, 'username') - ctx.username = 'fred' - self.m.StubOutWithMock(auth, 'authenticate') + ctx = self.create_context() auth.authenticate(ctx).AndReturn(True) - s = {} - s['name'] = stack.name - s['raw_template_id'] = new_rt.id - s['user_creds_id'] = new_creds.id - s['username'] = ct['username'] - new_s = db_api.stack_create(ctx, s) - stack.id = new_s.id - instances.Instance.nova().AndReturn(self.fc) + stack = self.get_wordpress_stack('test_stack_list', ctx) + self.m.ReplayAll() + stack.store() stack.create() - f = open("%s/WordPress_Single_Instance_gold.template" % self.path) - t = json.loads(f.read()) - params = {} - parameters = {} - t['Parameters']['KeyName']['Value'] = 'test' - stack = parser.Stack(ctx, 'test_stack_list', t, 0, params) - man = manager.EngineManager() - sl = man.list_stacks(ctx, params) + sl = man.list_stacks(ctx, {}) self.assertTrue(len(sl['stacks']) > 0) for s in sl['stacks']: - self.assertTrue(s['StackId'] > 0) + self.assertNotEqual(s['StackId'], None) self.assertNotEqual(s['TemplateDescription'].find('WordPress'), -1) + def test_stack_describe_all(self): + ctx = self.create_context('stack_describe_all') + auth.authenticate(ctx).AndReturn(True) + + stack = self.get_wordpress_stack('test_stack_desc_all', ctx) + + self.m.ReplayAll() + stack.store() + stack.create() + + man = manager.EngineManager() + sl = man.show_stack(ctx, None, {}) + + self.assertEqual(len(sl['stacks']), 1) + for s in sl['stacks']: + self.assertNotEqual(s['StackId'], None) + self.assertNotEqual(s['Description'].find('WordPress'), -1) + + def test_stack_describe_all_empty(self): + ctx = self.create_context('stack_describe_all_empty') + auth.authenticate(ctx).AndReturn(True) + + self.m.ReplayAll() + + man = manager.EngineManager() + sl = man.show_stack(ctx, None, {}) + + self.assertEqual(len(sl['stacks']), 0) + + def test_stack_describe_nonexistent(self): + ctx = self.create_context() + auth.authenticate(ctx).AndReturn(True) + + self.m.ReplayAll() + + man = manager.EngineManager() + sl = man.show_stack(ctx, 'wibble', {}) + + self.assertEqual(len(sl['stacks']), 0) + + def test_stack_describe(self): + ctx = self.create_context('stack_describe') + auth.authenticate(ctx).AndReturn(True) + + stack = self.get_wordpress_stack('test_stack_desc', ctx) + + self.m.ReplayAll() + stack.store() + stack.create() + + man = manager.EngineManager() + sl = man.show_stack(ctx, 'test_stack_desc', {}) + + self.assertTrue(len(sl['stacks']) > 0) + for s in sl['stacks']: + self.assertEqual(s['StackName'], 'test_stack_desc') + self.assertTrue('CreationTime' in s) + self.assertNotEqual(s['StackId'], None) + self.assertNotEqual(s['Description'].find('WordPress'), -1) + # allows testing of the test directly if __name__ == '__main__': sys.argv.append(__file__) diff --git a/heat/tests/test_validate.py b/heat/tests/test_validate.py index 14713cac84..967872d19b 100644 --- a/heat/tests/test_validate.py +++ b/heat/tests/test_validate.py @@ -215,8 +215,7 @@ class validateTest(unittest.TestCase): t = json.loads(test_template_volumeattach % 'vdq') self.m.StubOutWithMock(auth, 'authenticate') auth.authenticate(None).AndReturn(True) - params = {} - stack = parser.Stack(None, 'test_stack', t, 0, params) + stack = parser.Stack(None, 'test_stack', parser.Template(t)) self.m.StubOutWithMock(db_api, 'resource_get_by_name_and_stack') db_api.resource_get_by_name_and_stack(None, 'test_resource_name', @@ -224,14 +223,13 @@ class validateTest(unittest.TestCase): self.m.ReplayAll() volumeattach = stack.resources['MountPoint'] - assert(volumeattach.validate() is None) + self.assertTrue(volumeattach.validate() is None) def test_validate_volumeattach_invalid(self): t = json.loads(test_template_volumeattach % 'sda') self.m.StubOutWithMock(auth, 'authenticate') auth.authenticate(None).AndReturn(True) - params = {} - stack = parser.Stack(None, 'test_stack', t, 0, params) + stack = parser.Stack(None, 'test_stack', parser.Template(t)) self.m.StubOutWithMock(db_api, 'resource_get_by_name_and_stack') db_api.resource_get_by_name_and_stack(None, 'test_resource_name', diff --git a/heat/tests/test_waitcondition.py b/heat/tests/test_waitcondition.py index f881bb5628..6d84634425 100644 --- a/heat/tests/test_waitcondition.py +++ b/heat/tests/test_waitcondition.py @@ -7,12 +7,12 @@ import sys import nose import unittest -import mox from nose.plugins.attrib import attr from nose import with_setup import heat.db as db_api from heat.engine import parser +from heat.common import context logger = logging.getLogger('test_waitcondition') @@ -41,32 +41,15 @@ test_template_waitcondition = ''' @attr(speed='slow') class stacksTest(unittest.TestCase): def setUp(self): - self.m = mox.Mox() self.greenpool = eventlet.GreenPool() - def tearDown(self): - self.m.UnsetStubs() - def create_stack(self, stack_name, temp, params): - stack = parser.Stack(None, stack_name, temp, 0, params) - - rt = {} - rt['template'] = temp - rt['StackName'] = stack_name - new_rt = db_api.raw_template_create(None, rt) - - ct = {'username': 'fred', - 'password': 'mentions_fruit'} - new_creds = db_api.user_creds_create(ct) - - s = {} - s['name'] = stack_name - s['raw_template_id'] = new_rt.id - s['user_creds_id'] = new_creds.id - s['username'] = ct['username'] - new_s = db_api.stack_create(None, s) - stack.id = new_s.id + template = parser.Template(temp) + parameters = parser.Parameters(stack_name, template, params) + stack = parser.Stack(context.get_admin_context(), stack_name, + template, parameters) + stack.store() return stack def test_post_success_to_handle(self): @@ -74,7 +57,6 @@ class stacksTest(unittest.TestCase): t = json.loads(test_template_waitcondition) stack = self.create_stack('test_stack', t, params) - self.m.ReplayAll() self.greenpool.spawn_n(stack.create) eventlet.sleep(1) self.assertEqual(stack.resources['WaitForTheHandle'].state,