From 3f2f6361218a1764ff0e8fa6cd698104e8435c36 Mon Sep 17 00:00:00 2001 From: Meena Date: Tue, 22 Sep 2015 17:38:16 +0530 Subject: [PATCH] Added support for TOSCA group and group types Implemented groups and group types in definition template and their validations in topology template. Added unittestcases for the groups section. Partially implements: bp tosca-groups Change-Id: I62fa876c8156db3da0377205b66250c598282b9c --- toscaparser/common/exception.py | 4 + .../elements/TOSCA_definition_1_0.yaml | 12 ++ toscaparser/elements/entity_type.py | 1 + toscaparser/elements/grouptype.py | 86 ++++++++++++ toscaparser/entity_template.py | 5 + toscaparser/groups.py | 17 ++- .../tests/data/groups/definitions.yaml | 10 ++ .../data/groups/tosca_group_template.yaml | 54 +++++++ .../data/policies/tosca_policy_template.yaml | 2 +- .../data/topology_template/subsystem.yaml | 3 +- toscaparser/tests/test_topology_template.py | 4 +- toscaparser/tests/test_toscadef.py | 6 + toscaparser/tests/test_toscatpl.py | 2 +- toscaparser/tests/test_toscatplvalidation.py | 132 ++++++++++++++++++ toscaparser/topology_template.py | 45 ++++-- toscaparser/tosca_template.py | 6 +- 16 files changed, 360 insertions(+), 29 deletions(-) create mode 100644 toscaparser/elements/grouptype.py create mode 100644 toscaparser/tests/data/groups/definitions.yaml create mode 100644 toscaparser/tests/data/groups/tosca_group_template.yaml diff --git a/toscaparser/common/exception.py b/toscaparser/common/exception.py index 8c09f484..96145d1a 100644 --- a/toscaparser/common/exception.py +++ b/toscaparser/common/exception.py @@ -117,6 +117,10 @@ class ToscaExtAttributeError(TOSCAException): '"%(attrs)s" defined.') +class InvalidGroupTargetException(TOSCAException): + msg_fmt = _('"%(message)s"') + + class ExceptionCollector(object): exceptions = [] diff --git a/toscaparser/elements/TOSCA_definition_1_0.yaml b/toscaparser/elements/TOSCA_definition_1_0.yaml index 3cb74b96..55d6a55b 100644 --- a/toscaparser/elements/TOSCA_definition_1_0.yaml +++ b/toscaparser/elements/TOSCA_definition_1_0.yaml @@ -778,3 +778,15 @@ tosca.policies.Performance: derived_from: tosca.policies.Root description: The TOSCA Policy Type definition that is used to declare performance requirements for TOSCA nodes or groups of nodes. + +########################################################################## + # Group Type. + # Group Type represents logical grouping of TOSCA nodes that have an + # implied membership relationship and may need to be orchestrated or + # managed together to achieve some result. +########################################################################## +tosca.groups.Root: + description: The TOSCA Group Type all other TOSCA Group Types derive from + interfaces: + Standard: + type: tosca.interfaces.node.lifecycle.Standard diff --git a/toscaparser/elements/entity_type.py b/toscaparser/elements/entity_type.py index a8a4df00..4211067f 100644 --- a/toscaparser/elements/entity_type.py +++ b/toscaparser/elements/entity_type.py @@ -52,6 +52,7 @@ class EntityType(object): INTERFACE_PREFIX = 'tosca.interfaces.' ARTIFACT_PREFIX = 'tosca.artifacts.' POLICY_PREFIX = 'tosca.policies.' + GROUP_PREFIX = 'tosca.groups.' # currently the data types are defined only for network # but may have changes in the future. DATATYPE_PREFIX = 'tosca.datatypes.network.' diff --git a/toscaparser/elements/grouptype.py b/toscaparser/elements/grouptype.py new file mode 100644 index 00000000..ec5571c2 --- /dev/null +++ b/toscaparser/elements/grouptype.py @@ -0,0 +1,86 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from toscaparser.common.exception import ExceptionCollector +from toscaparser.common.exception import InvalidTypeError +from toscaparser.common.exception import UnknownFieldError +from toscaparser.elements.statefulentitytype import StatefulEntityType + + +class GroupType(StatefulEntityType): + '''TOSCA built-in group type.''' + + SECTIONS = (DERIVED_FROM, VERSION, METADATA, DESCRIPTION, PROPERTIES, + MEMBERS, INTERFACES) = \ + ("derived_from", "version", "metadata", "description", + "properties", "members", "interfaces") + + def __init__(self, grouptype, custom_def=None): + super(GroupType, self).__init__(grouptype, self.GROUP_PREFIX, + custom_def) + self.custom_def = custom_def + self.grouptype = grouptype + self._validate_fields() + self.group_description = None + if self.DESCRIPTION in self.defs: + self.group_description = self.defs[self.DESCRIPTION] + + self.group_version = None + if self.VERSION in self.defs: + self.group_version = self.defs[self.VERSION] + + self.group_properties = None + if self.PROPERTIES in self.defs: + self.group_properties = self.defs[self.PROPERTIES] + + self.group_members = None + if self.MEMBERS in self.defs: + self.group_members = self.defs[self.MEMBERS] + + if self.METADATA in self.defs: + self.meta_data = self.defs[self.METADATA] + self._validate_metadata(self.meta_data) + + @property + def description(self): + return self.group_description + + @property + def version(self): + return self.group_version + + @property + def interfaces(self): + return self.get_value(self.INTERFACES) + + def _validate_fields(self): + if self.defs: + for name in self.defs.keys(): + if name not in self.SECTIONS: + ExceptionCollector.appendException( + UnknownFieldError(what='Group Type %s' + % self.grouptype, field=name)) + + def _validate_metadata(self, meta_data): + if not meta_data.get('type') in ['map', 'tosca:map']: + ExceptionCollector.appendException( + InvalidTypeError(what='"%s" defined in group for ' + 'metadata' % (meta_data.get('type')))) + for entry_schema, entry_schema_type in meta_data.items(): + if isinstance(entry_schema_type, dict) and not \ + entry_schema_type.get('type') == 'string': + ExceptionCollector.appendException( + InvalidTypeError(what='"%s" defined in group for ' + 'metadata "%s"' + % (entry_schema_type.get('type'), + entry_schema))) diff --git a/toscaparser/entity_template.py b/toscaparser/entity_template.py index aaf02e5d..b25bfbea 100644 --- a/toscaparser/entity_template.py +++ b/toscaparser/entity_template.py @@ -15,6 +15,7 @@ from toscaparser.common.exception import ExceptionCollector from toscaparser.common.exception import MissingRequiredFieldError from toscaparser.common.exception import UnknownFieldError from toscaparser.common.exception import ValidationError +from toscaparser.elements.grouptype import GroupType from toscaparser.elements.interfaces import InterfacesDef from toscaparser.elements.nodetype import NodeType from toscaparser.elements.policytype import PolicyType @@ -67,6 +68,10 @@ class EntityTemplate(object): ValidationError(msg)) self.type_definition = PolicyType(type, custom_def) + if entity_name == 'group_type': + type = self.entity_tpl.get('type') + self.type_definition = GroupType(type, custom_def) \ + if type is not None else None self._properties = None self._interfaces = None self._requirements = None diff --git a/toscaparser/groups.py b/toscaparser/groups.py index 9f23cf38..5fd5dece 100644 --- a/toscaparser/groups.py +++ b/toscaparser/groups.py @@ -10,7 +10,6 @@ # License for the specific language governing permissions and limitations # under the License. - from toscaparser.common.exception import ExceptionCollector from toscaparser.common.exception import UnknownFieldError from toscaparser.entity_template import EntityTemplate @@ -18,35 +17,35 @@ from toscaparser.utils import validateutils SECTIONS = (TYPE, METADATA, DESCRIPTION, PROPERTIES, TARGETS, INTERFACES) = \ ('type', 'metadata', 'description', - 'properties', 'targets', 'interfaces') + 'properties', 'members', 'interfaces') class Group(EntityTemplate): - def __init__(self, name, group_templates, member_nodes): + def __init__(self, name, group_templates, member_nodes, custom_defs=None): super(Group, self).__init__(name, group_templates, 'group_type', - None) + custom_defs) self.name = name self.tpl = group_templates self.meta_data = None if self.METADATA in self.tpl: self.meta_data = self.tpl.get(self.METADATA) validateutils.validate_map(self.meta_data) - self.members = member_nodes + self.member_nodes = member_nodes self._validate_keys() @property - def targets(self): - return self.tpl.get('targets') + def members(self): + return self.entity_tpl.get('members') @property def description(self): return self.entity_tpl.get('description') - def get_members(self): - return self.members + def get_member_nodes(self): + return self.member_nodes def _validate_keys(self): for key in self.entity_tpl.keys(): diff --git a/toscaparser/tests/data/groups/definitions.yaml b/toscaparser/tests/data/groups/definitions.yaml new file mode 100644 index 00000000..40c1d8b7 --- /dev/null +++ b/toscaparser/tests/data/groups/definitions.yaml @@ -0,0 +1,10 @@ +tosca_definitions_version: tosca_simple_yaml_1_0 + +group_types: + mycompany.mytypes.groups.placement: + description: My company's group type for placing nodes of type Compute + members: [ tosca.nodes.Compute ] + metadata: + type: map + entry_schema: + type: string diff --git a/toscaparser/tests/data/groups/tosca_group_template.yaml b/toscaparser/tests/data/groups/tosca_group_template.yaml new file mode 100644 index 00000000..0e94240e --- /dev/null +++ b/toscaparser/tests/data/groups/tosca_group_template.yaml @@ -0,0 +1,54 @@ +tosca_definitions_version: tosca_simple_yaml_1_0 + +description: > + Service template with topology_template, act as a nested system inside another system. + +imports: + - definitions.yaml + +topology_template: + description: Template of a database including its hosting stack. + + inputs: + mq_server_ip: + type: string + description: IP address of the message queuing server to receive messages from. + receiver_port: + type: string + description: Port to be used for receiving messages. + my_cpus: + type: integer + description: Number of CPUs for the server. + constraints: + - valid_values: [ 1, 2, 4, 8 ] + + node_templates: + websrv: + type: tosca.nodes.WebServer + capabilities: + data_endpoint: + properties: + port_name: { get_input: receiver_port } + requirements: + - host: + node: server + + server: + type: tosca.nodes.Compute + capabilities: + host: + properties: + disk_size: 10 GB + num_cpus: { get_input: my_cpus } + mem_size: 4096 MB + os: + properties: + architecture: x86_64 + type: Linux + distribution: Ubuntu + version: 14.04 + + groups: + webserver_group: + type: mycompany.mytypes.groups.placement + members: [ websrv, server ] diff --git a/toscaparser/tests/data/policies/tosca_policy_template.yaml b/toscaparser/tests/data/policies/tosca_policy_template.yaml index 3a53bf13..de435631 100644 --- a/toscaparser/tests/data/policies/tosca_policy_template.yaml +++ b/toscaparser/tests/data/policies/tosca_policy_template.yaml @@ -43,7 +43,7 @@ topology_template: groups: webserver_group: - targets: [ my_server_1, my_server_2 ] + members: [ my_server_1, my_server_2 ] type: tosca.groups.Root metadata: { user1: 1008, user2: 1002 } diff --git a/toscaparser/tests/data/topology_template/subsystem.yaml b/toscaparser/tests/data/topology_template/subsystem.yaml index 0efbbe86..b27e6981 100644 --- a/toscaparser/tests/data/topology_template/subsystem.yaml +++ b/toscaparser/tests/data/topology_template/subsystem.yaml @@ -79,6 +79,5 @@ topology_template: groups: webserver_group: - targets: [ websrv, server ] + members: [ websrv, server ] type: tosca.groups.Root - diff --git a/toscaparser/tests/test_topology_template.py b/toscaparser/tests/test_topology_template.py index aea4822f..b72c7e1a 100644 --- a/toscaparser/tests/test_topology_template.py +++ b/toscaparser/tests/test_topology_template.py @@ -145,8 +145,8 @@ class TopologyTemplateTest(TestCase): def test_groups(self): group = self.topo.groups[0] self.assertEqual('webserver_group', group.name) - self.assertEqual(['websrv', 'server'], group.targets) - for node in group.members: + self.assertEqual(['websrv', 'server'], group.members) + for node in group.get_member_nodes(): if node.name == 'server': '''Test property value''' props = node.get_properties() diff --git a/toscaparser/tests/test_toscadef.py b/toscaparser/tests/test_toscadef.py index 38d08d7a..89871b57 100644 --- a/toscaparser/tests/test_toscadef.py +++ b/toscaparser/tests/test_toscadef.py @@ -13,6 +13,7 @@ from toscaparser.common import exception from toscaparser.elements.artifacttype import ArtifactTypeDef from toscaparser.elements.entity_type import EntityType +from toscaparser.elements.grouptype import GroupType import toscaparser.elements.interfaces as ifaces from toscaparser.elements.nodetype import NodeType from toscaparser.elements.policytype import PolicyType @@ -40,6 +41,7 @@ policy_placement_type = PolicyType('tosca.policies.Placement') policy_scaling_type = PolicyType('tosca.policies.Scaling') policy_update_type = PolicyType('tosca.policies.Update') policy_performance_type = PolicyType('tosca.policies.Performance') +group_type = GroupType('tosca.groups.Root') class ToscaDefTest(TestCase): @@ -56,6 +58,10 @@ class ToscaDefTest(TestCase): self.assertEqual(network_port_type.parent_type.type, "tosca.nodes.Root") + def test_group(self): + self.assertEqual(group_type.type, "tosca.groups.Root") + self.assertIn(ifaces.LIFECYCLE_SHORTNAME, group_type.interfaces) + def test_capabilities(self): self.assertEqual( sorted(['tosca.capabilities.Container', diff --git a/toscaparser/tests/test_toscatpl.py b/toscaparser/tests/test_toscatpl.py index 7dac5c1a..12330024 100644 --- a/toscaparser/tests/test_toscatpl.py +++ b/toscaparser/tests/test_toscatpl.py @@ -650,7 +650,7 @@ class ToscaTemplateTest(TestCase): self.assertEqual(['webserver_group'], policy.targets) self.assertEqual('groups', policy.get_targets_type()) group = policy.get_targets_list()[0] - for node in group.get_members(): + for node in group.get_member_nodes(): if node.name == 'my_server_2': '''Test property value''' props = node.get_properties() diff --git a/toscaparser/tests/test_toscatplvalidation.py b/toscaparser/tests/test_toscatplvalidation.py index 74ddf304..84385154 100644 --- a/toscaparser/tests/test_toscatplvalidation.py +++ b/toscaparser/tests/test_toscatplvalidation.py @@ -21,6 +21,7 @@ from toscaparser.parameters import Output from toscaparser.policy import Policy from toscaparser.relationship_template import RelationshipTemplate from toscaparser.tests.base import TestCase +from toscaparser.topology_template import TopologyTemplate from toscaparser.tosca_template import ToscaTemplate from toscaparser.utils.gettextutils import _ @@ -336,6 +337,137 @@ custom_types/wordpress.yaml 'to verify valid values.'), err.__str__()) + def test_groups(self): + tpl_snippet = ''' + node_templates: + server: + type: tosca.nodes.Compute + requirements: + - log_endpoint: + capability: log_endpoint + + mysql_dbms: + type: tosca.nodes.DBMS + properties: + root_password: aaa + port: 3376 + + groups: + webserver_group: + type: tosca.groups.Root + members: [ server, mysql_dbms ] + ''' + tpl = (toscaparser.utils.yamlparser.simple_parse(tpl_snippet)) + TopologyTemplate(tpl, None) + + def test_groups_with_missing_required_field(self): + tpl_snippet = ''' + node_templates: + server: + type: tosca.nodes.Compute + requirements: + - log_endpoint: + capability: log_endpoint + + mysql_dbms: + type: tosca.nodes.DBMS + properties: + root_password: aaa + port: 3376 + + groups: + webserver_group: + members: ['server', 'mysql_dbms'] + ''' + tpl = (toscaparser.utils.yamlparser.simple_parse(tpl_snippet)) + err = self.assertRaises(exception.MissingRequiredFieldError, + TopologyTemplate, tpl, None) + expectedmessage = _('Template "webserver_group" is missing ' + 'required field "type".') + self.assertEqual(expectedmessage, err.__str__()) + + def test_groups_with_unknown_target(self): + tpl_snippet = ''' + node_templates: + server: + type: tosca.nodes.Compute + requirements: + - log_endpoint: + capability: log_endpoint + + mysql_dbms: + type: tosca.nodes.DBMS + properties: + root_password: aaa + port: 3376 + + groups: + webserver_group: + type: tosca.groups.Root + members: [ serv, mysql_dbms ] + ''' + tpl = (toscaparser.utils.yamlparser.simple_parse(tpl_snippet)) + expectedmessage = _('"Target member "serv" is not found in ' + 'node_templates"') + err = self.assertRaises(exception.InvalidGroupTargetException, + TopologyTemplate, tpl, None) + self.assertEqual(expectedmessage, err.__str__()) + + def test_groups_with_repeated_targets(self): + tpl_snippet = ''' + node_templates: + server: + type: tosca.nodes.Compute + requirements: + - log_endpoint: + capability: log_endpoint + + mysql_dbms: + type: tosca.nodes.DBMS + properties: + root_password: aaa + port: 3376 + + groups: + webserver_group: + type: tosca.groups.Root + members: [ server, server, mysql_dbms ] + ''' + tpl = (toscaparser.utils.yamlparser.simple_parse(tpl_snippet)) + expectedmessage = _('"Member nodes ' + '"[\'server\', \'server\', \'mysql_dbms\']" ' + 'should be >= 1 and not repeated"') + err = self.assertRaises(exception.InvalidGroupTargetException, + TopologyTemplate, tpl, None) + self.assertEqual(expectedmessage, err.__str__()) + + def test_groups_with_only_one_target(self): + tpl_snippet = ''' + node_templates: + server: + type: tosca.nodes.Compute + requirements: + - log_endpoint: + capability: log_endpoint + + mysql_dbms: + type: tosca.nodes.DBMS + properties: + root_password: aaa + port: 3376 + + groups: + webserver_group: + type: tosca.groups.Root + members: [] + ''' + tpl = (toscaparser.utils.yamlparser.simple_parse(tpl_snippet)) + expectedmessage = _('"Member nodes "[]" should be >= 1 ' + 'and not repeated"') + err = self.assertRaises(exception.InvalidGroupTargetException, + TopologyTemplate, tpl, None) + self.assertEqual(expectedmessage, err.__str__()) + def _custom_types(self): custom_types = {} def_file = os.path.join( diff --git a/toscaparser/topology_template.py b/toscaparser/topology_template.py index 972f3a24..f8f88754 100644 --- a/toscaparser/topology_template.py +++ b/toscaparser/topology_template.py @@ -14,6 +14,7 @@ import logging from toscaparser.common import exception +from toscaparser.dataentity import DataEntity from toscaparser import functions from toscaparser.groups import Group from toscaparser.nodetemplate import NodeTemplate @@ -22,6 +23,7 @@ from toscaparser.parameters import Output from toscaparser.policy import Policy from toscaparser.relationship_template import RelationshipTemplate from toscaparser.tpl_relationship_graph import ToscaGraph +from toscaparser.utils.gettextutils import _ # Topology template key names @@ -108,11 +110,11 @@ class TopologyTemplate(object): target_list = policy_tpl.get('targets') if target_list and len(target_list) >= 1: target_objects = [] - targets_type = "node_templates" - target_objects = self._get_group_members(target_list) + targets_type = "groups" + target_objects = self._get_policy_groups(target_list) if not target_objects: - target_objects = self._get_policy_groups(target_list) - targets_type = "groups" + targets_type = "node_templates" + target_objects = self._get_group_members(target_list) policyObj = Policy(policy_name, policy_tpl, target_objects, targets_type, self.custom_defs) @@ -121,18 +123,28 @@ class TopologyTemplate(object): def _groups(self): groups = [] + member_nodes = None for group_name, group_tpl in self._tpl_groups().items(): - member_names = group_tpl.get('targets') - if member_names and len(member_names) > 1: - group = Group(group_name, group_tpl, - self._get_group_members(member_names)) - groups.append(group) - else: - exception.ExceptionCollector.appendException(ValueError) + member_names = group_tpl.get('members') + if member_names is not None: + DataEntity.validate_datatype('list', member_names) + if len(member_names) < 1 or \ + len(member_names) != len(set(member_names)): + exception.ExceptionCollector.appendException( + exception.InvalidGroupTargetException( + message=_('Member nodes "%s" should be >= 1 ' + 'and not repeated') % member_names)) + else: + member_nodes = self._get_group_members(member_names) + group = Group(group_name, group_tpl, + member_nodes, + self.custom_defs) + groups.append(group) return groups def _get_group_members(self, member_names): member_nodes = [] + self._validate_group_members(member_names) for member in member_names: for node in self.nodetemplates: if node.name == member: @@ -147,6 +159,17 @@ class TopologyTemplate(object): member_groups.append(group) return member_groups + def _validate_group_members(self, members): + node_names = [] + for node in self.nodetemplates: + node_names.append(node.name) + for member in members: + if member not in node_names: + exception.ExceptionCollector.appendException( + exception.InvalidGroupTargetException( + message=_('Target member "%s" is not found in ' + 'node_templates') % member)) + # topology template can act like node template # it is exposed by substitution_mappings. def nodetype(self): diff --git a/toscaparser/tosca_template.py b/toscaparser/tosca_template.py index 8d7b22d4..4591a14b 100644 --- a/toscaparser/tosca_template.py +++ b/toscaparser/tosca_template.py @@ -35,13 +35,13 @@ SECTIONS = (DEFINITION_VERSION, DEFAULT_NAMESPACE, TEMPLATE_NAME, DESCRIPTION, IMPORTS, DSL_DEFINITIONS, NODE_TYPES, RELATIONSHIP_TYPES, RELATIONSHIP_TEMPLATES, CAPABILITY_TYPES, ARTIFACT_TYPES, DATATYPE_DEFINITIONS, - POLICY_TYPES) = \ + POLICY_TYPES, GROUP_TYPES) = \ ('tosca_definitions_version', 'tosca_default_namespace', 'template_name', 'topology_template', 'template_author', 'template_version', 'description', 'imports', 'dsl_definitions', 'node_types', 'relationship_types', 'relationship_templates', 'capability_types', 'artifact_types', 'datatype_definitions', - 'policy_types') + 'policy_types', 'group_types') # Sections that are specific to individual template definitions SPECIAL_SECTIONS = (METADATA) = ('metadata') @@ -145,7 +145,7 @@ class ToscaTemplate(object): def _get_all_custom_defs(self, imports=None): types = [IMPORTS, NODE_TYPES, CAPABILITY_TYPES, RELATIONSHIP_TYPES, - DATATYPE_DEFINITIONS, POLICY_TYPES] + DATATYPE_DEFINITIONS, POLICY_TYPES, GROUP_TYPES] custom_defs_final = {} custom_defs = self._get_custom_types(types, imports) if custom_defs: