From e37c92327ad983df36ba3c6ae4fb7d2c0106a831 Mon Sep 17 00:00:00 2001 From: Yoshiro Watanabe Date: Tue, 27 Aug 2024 09:08:01 +0000 Subject: [PATCH] Updated TOSCA1.3 support in tosca-parser tosca-parser has had some updates to bring it into partial compliance with the TOSCA Simple Profile for YAML version 1.3. The implementation for the following updated items has been added: * 'attributes' was added to the keynames of the Group definition * 'substitution_filter' was added to the keynames of the Substitution mapping * Added tosca.artifacts.template * Added tosca.artifacts.template.Jinja2 * Added tosca.artifacts.template.Twig Change-Id: Iabd748640013e0108d4ac7b4b4fb8a48c412330c --- ...tution_mappings_error_capability_type.yaml | 25 ++++++ ...bstitution_mappings_error_node_filter.yaml | 25 ++++++ ...tution_mappings_error_node_filter_key.yaml | 25 ++++++ ...titution_mappings_error_property_type.yaml | 25 ++++++ .../test_substitution_mappings_section.yaml | 51 +++++++++++ ...inition_1_0.yaml => TOSCA_definition.yaml} | 18 +++- toscaparser/elements/entity_type.py | 2 +- toscaparser/elements/grouptype.py | 4 +- toscaparser/entity_template.py | 9 ++ toscaparser/groups.py | 5 +- toscaparser/substitution_mappings.py | 84 +++++++++++++++++- toscaparser/tests/test_topology_template.py | 88 +++++++++++++++++++ toscaparser/tests/test_toscadef.py | 49 +++++++++++ toscaparser/tests/test_toscatplvalidation.py | 26 ++++++ 14 files changed, 427 insertions(+), 9 deletions(-) create mode 100644 samples/tests/data/topology_template/validate/test_substitution_mappings_error_capability_type.yaml create mode 100644 samples/tests/data/topology_template/validate/test_substitution_mappings_error_node_filter.yaml create mode 100644 samples/tests/data/topology_template/validate/test_substitution_mappings_error_node_filter_key.yaml create mode 100644 samples/tests/data/topology_template/validate/test_substitution_mappings_error_property_type.yaml create mode 100644 samples/tests/data/topology_template/validate/test_substitution_mappings_section.yaml rename toscaparser/elements/{TOSCA_definition_1_0.yaml => TOSCA_definition.yaml} (98%) diff --git a/samples/tests/data/topology_template/validate/test_substitution_mappings_error_capability_type.yaml b/samples/tests/data/topology_template/validate/test_substitution_mappings_error_capability_type.yaml new file mode 100644 index 00000000..eb941ed8 --- /dev/null +++ b/samples/tests/data/topology_template/validate/test_substitution_mappings_error_capability_type.yaml @@ -0,0 +1,25 @@ +tosca_definitions_version: tosca_simple_yaml_1_3 + +topology_template: + inputs: + my_property: + type: string + + substitution_mappings: + node_type: my.company.MyService + substitution_filter: + node_filter: + properties: + - my_property: 'property_value' + capabilities: + capability_name_or_type_1: + properties: + - cap_1_property_filter_def_1 + - cap_1_property_filter_def_2 + +node_types: + my.company.MyService: + derived_from: tosca.nodes.Root + properties: + my_property: + type: string \ No newline at end of file diff --git a/samples/tests/data/topology_template/validate/test_substitution_mappings_error_node_filter.yaml b/samples/tests/data/topology_template/validate/test_substitution_mappings_error_node_filter.yaml new file mode 100644 index 00000000..20bc36c0 --- /dev/null +++ b/samples/tests/data/topology_template/validate/test_substitution_mappings_error_node_filter.yaml @@ -0,0 +1,25 @@ +tosca_definitions_version: tosca_simple_yaml_1_3 + +topology_template: + inputs: + my_property: + type: string + + substitution_mappings: + node_type: my.company.MyService + substitution_filter: + node_filter_test: + properties: + - my_property: 'property_value' + capabilities: + - capability_name_or_type_1: + properties: + - cap_1_property_filter_def_1 + - cap_1_property_filter_def_2 + +node_types: + my.company.MyService: + derived_from: tosca.nodes.Root + properties: + my_property: + type: string \ No newline at end of file diff --git a/samples/tests/data/topology_template/validate/test_substitution_mappings_error_node_filter_key.yaml b/samples/tests/data/topology_template/validate/test_substitution_mappings_error_node_filter_key.yaml new file mode 100644 index 00000000..4f54f4a4 --- /dev/null +++ b/samples/tests/data/topology_template/validate/test_substitution_mappings_error_node_filter_key.yaml @@ -0,0 +1,25 @@ +tosca_definitions_version: tosca_simple_yaml_1_3 + +topology_template: + inputs: + my_property: + type: string + + substitution_mappings: + node_type: my.company.MyService + substitution_filter: + node_filter: + properties_test: + - my_property: 'property_value' + capabilities: + - capability_name_or_type_1: + properties: + - cap_1_property_filter_def_1 + - cap_1_property_filter_def_2 + +node_types: + my.company.MyService: + derived_from: tosca.nodes.Root + properties: + my_property: + type: string \ No newline at end of file diff --git a/samples/tests/data/topology_template/validate/test_substitution_mappings_error_property_type.yaml b/samples/tests/data/topology_template/validate/test_substitution_mappings_error_property_type.yaml new file mode 100644 index 00000000..f9e345f4 --- /dev/null +++ b/samples/tests/data/topology_template/validate/test_substitution_mappings_error_property_type.yaml @@ -0,0 +1,25 @@ +tosca_definitions_version: tosca_simple_yaml_1_3 + +topology_template: + inputs: + my_property: + type: string + + substitution_mappings: + node_type: my.company.MyService + substitution_filter: + node_filter: + properties: + my_property: 'property_value' + capabilities: + - capability_name_or_type_1: + properties: + - cap_1_property_filter_def_1 + - cap_1_property_filter_def_2 + +node_types: + my.company.MyService: + derived_from: tosca.nodes.Root + properties: + my_property: + type: string \ No newline at end of file diff --git a/samples/tests/data/topology_template/validate/test_substitution_mappings_section.yaml b/samples/tests/data/topology_template/validate/test_substitution_mappings_section.yaml new file mode 100644 index 00000000..59a7fd3e --- /dev/null +++ b/samples/tests/data/topology_template/validate/test_substitution_mappings_section.yaml @@ -0,0 +1,51 @@ +tosca_definitions_version: tosca_simple_yaml_1_3 + +topology_template: + inputs: + my_property: + type: string + + substitution_mappings: + node_type: my.company.MyService + interfaces: + Standard: + start: my_start_node.start + attributes: + receiver_ip: my_property + substitution_filter: + node_filter: + properties: + - my_property: 'property_value' + capabilities: + - capability_name_or_type_1: + properties: + - cap_1_property_filter_def_1 + - cap_1_property_filter_def_2 + + node_templates: + my_start_node: + type: my.company.MyServiceCompute + properties: + my_property: { get_input: my_property } + interfaces: + Standard: + start: + implementation: scripts/start.sh + inputs: + my_property: { get_property: [ SELF, my_property ] } + +node_types: + my.company.MyService: + derived_from: tosca.nodes.Root + properties: + my_property: + type: string + attributes: + receiver_ip: + type: string + + my.company.MyServiceCompute: + derived_from: tosca.nodes.Compute + properties: + my_property: + type: string diff --git a/toscaparser/elements/TOSCA_definition_1_0.yaml b/toscaparser/elements/TOSCA_definition.yaml similarity index 98% rename from toscaparser/elements/TOSCA_definition_1_0.yaml rename to toscaparser/elements/TOSCA_definition.yaml index 1f676665..3a4618a7 100644 --- a/toscaparser/elements/TOSCA_definition_1_0.yaml +++ b/toscaparser/elements/TOSCA_definition.yaml @@ -12,10 +12,9 @@ ########################################################################## # The content of this file reflects TOSCA Simple Profile in YAML version -# 1.0.0. It describes the definition for TOSCA types including Node Type, +# 1.0.3. It describes the definition for TOSCA types including Node Type, # Relationship Type, Capability Type and Interfaces. ########################################################################## -tosca_definitions_version: tosca_simple_yaml_1_0 ########################################################################## # Node Type. @@ -918,6 +917,21 @@ artifact_types: mime_type: application/octet-stream file_ext: [ qcow2 ] + # Added in TOSCA1.3 + tosca.artifacts.template: + derived_from: tosca.artifacts.Root + description: TOSCA base type for template type artifacts + + # Added in TOSCA1.3 + tosca.artifacts.template.Jinja2: + derived_from: tosca.artifacts.template + description: Jinja2 template file + + # Added in TOSCA1.3 + tosca.artifacts.template.Twig: + derived_from: tosca.artifacts.template + description: Twig template file + ########################################################################## # Policy Type. # TOSCA Policy Types represent logical grouping of TOSCA nodes that have diff --git a/toscaparser/elements/entity_type.py b/toscaparser/elements/entity_type.py index 5c672768..a23d630d 100644 --- a/toscaparser/elements/entity_type.py +++ b/toscaparser/elements/entity_type.py @@ -37,7 +37,7 @@ class EntityType(object): '''TOSCA definition file.''' TOSCA_DEF_FILE = os.path.join( os.path.dirname(os.path.abspath(__file__)), - "TOSCA_definition_1_0.yaml") + "TOSCA_definition.yaml") loader = toscaparser.utils.yamlparser.load_yaml diff --git a/toscaparser/elements/grouptype.py b/toscaparser/elements/grouptype.py index b85266fa..4406be0a 100644 --- a/toscaparser/elements/grouptype.py +++ b/toscaparser/elements/grouptype.py @@ -23,9 +23,9 @@ class GroupType(StatefulEntityType): # In TOSCA1.3, the section name 'interfaces' is not defined for group type. # It will be ignored if the 'interfaces' is defined as TOSCA1.3. SECTIONS = (DERIVED_FROM, VERSION, METADATA, DESCRIPTION, PROPERTIES, - MEMBERS, INTERFACES) = \ + MEMBERS, INTERFACES, ATTRIBUTES) = \ ("derived_from", "version", "metadata", "description", - "properties", "members", "interfaces") + "properties", "members", "interfaces", "attributes") def __init__(self, grouptype, custom_def=None): super(GroupType, self).__init__(grouptype, self.GROUP_PREFIX, diff --git a/toscaparser/entity_template.py b/toscaparser/entity_template.py index 7305ec16..20beacbc 100644 --- a/toscaparser/entity_template.py +++ b/toscaparser/entity_template.py @@ -77,6 +77,7 @@ class EntityTemplate(object): self._interfaces = None self._requirements = None self._capabilities = None + self._attributes = None @property def type(self): @@ -96,6 +97,14 @@ class EntityTemplate(object): self.entity_tpl) or [] return self._requirements + @property + def attributes(self): + if self._attributes is None: + self._attributes = self.type_definition.get_value( + self.ATTRIBUTES, + self.entity_tpl) or [] + return self._attributes + def get_properties_objects(self): '''Return properties objects for this template.''' if self._properties is None: diff --git a/toscaparser/groups.py b/toscaparser/groups.py index a50eab3f..b92d29b8 100644 --- a/toscaparser/groups.py +++ b/toscaparser/groups.py @@ -17,9 +17,10 @@ from toscaparser.utils import validateutils # In TOSCA1.3, the section name 'interfaces' is not defined for groups. # It will be ignored if the 'interfaces' is defined as TOSCA1.3. -SECTIONS = (TYPE, METADATA, DESCRIPTION, PROPERTIES, MEMBERS, INTERFACES) = \ +SECTIONS = (TYPE, METADATA, DESCRIPTION, PROPERTIES, MEMBERS, INTERFACES, + ATTRIBUTES) = \ ('type', 'metadata', 'description', - 'properties', 'members', 'interfaces') + 'properties', 'members', 'interfaces', 'attributes') class Group(EntityTemplate): diff --git a/toscaparser/substitution_mappings.py b/toscaparser/substitution_mappings.py index 95dc4e21..1e1d56a2 100644 --- a/toscaparser/substitution_mappings.py +++ b/toscaparser/substitution_mappings.py @@ -17,6 +17,7 @@ from toscaparser.common.exception import InvalidNodeTypeError from toscaparser.common.exception import MissingDefaultValueError from toscaparser.common.exception import MissingRequiredFieldError from toscaparser.common.exception import MissingRequiredInputError +from toscaparser.common.exception import TypeMismatchError from toscaparser.common.exception import UnknownFieldError from toscaparser.common.exception import UnknownOutputError from toscaparser.elements.nodetype import NodeType @@ -32,8 +33,10 @@ class SubstitutionMappings(object): implementation of a Node type. ''' - SECTIONS = (NODE_TYPE, REQUIREMENTS, CAPABILITIES, PROPERTIES) = \ - ('node_type', 'requirements', 'capabilities', 'properties') + SECTIONS = (NODE_TYPE, REQUIREMENTS, CAPABILITIES, PROPERTIES, + ATTRIBUTES, INTERFACES, SUBSTITUTION_FILTER) = \ + ('node_type', 'requirements', 'capabilities', 'properties', + 'attributes', 'interfaces', 'substitution_filter') OPTIONAL_OUTPUTS = ['tosca_id', 'tosca_name', 'state'] @@ -77,6 +80,18 @@ class SubstitutionMappings(object): def properties(self): return self.sub_mapping_def.get(self.PROPERTIES) + @property + def attributes(self): + return self.sub_mapping_def.get(self.ATTRIBUTES) + + @property + def interfaces(self): + return self.sub_mapping_def.get(self.INTERFACES) + + @property + def substitution_filter(self): + return self.sub_mapping_def.get(self.SUBSTITUTION_FILTER) + @property def node_definition(self): return NodeType(self.node_type, self.custom_defs) @@ -92,6 +107,9 @@ class SubstitutionMappings(object): self._validate_requirements() self._validate_properties() self._validate_outputs() + self._validate_attributes() + self._validate_interfaces() + self._validate_substitution_filter() def _validate_keys(self): """validate the keys of substitution mappings.""" @@ -234,3 +252,65 @@ class SubstitutionMappings(object): where=_('SubstitutionMappings with node_type ') + self.node_type, output_name=output.name)) + + def _validate_attributes(self): + """validate the attributes of substitution mappings.""" + + # The attributes must be in node template wchich be mapped. + tpls_attributes = self.sub_mapping_def.get(self.ATTRIBUTES) + node_attributes = self.sub_mapped_node_template.attributes \ + if self.sub_mapped_node_template else None + for req in node_attributes if node_attributes else []: + if (tpls_attributes and + req not in list(tpls_attributes.keys())): + pass + # ExceptionCollector.appendException( + # UnknownFieldError(what='SubstitutionMappings', + # field=req)) + + def _validate_interfaces(self): + """validate the interfaces of substitution mappings.""" + + # The interfaces must be in node template wchich be mapped. + tpls_interfaces = self.sub_mapping_def.get(self.INTERFACES) + node_interfaces = self.sub_mapped_node_template.interfaces \ + if self.sub_mapped_node_template else None + for req in node_interfaces if node_interfaces else []: + if (tpls_interfaces and + req not in list(tpls_interfaces.keys())): + pass + # ExceptionCollector.appendException( + # UnknownFieldError(what='SubstitutionMappings', + # field=req)) + + def _validate_substitution_filter(self): + tpls_filter = self.sub_mapping_def.get( + self.SUBSTITUTION_FILTER) + if tpls_filter: + for key, value in tpls_filter.items(): + if key != 'node_filter': + ExceptionCollector.appendException( + UnknownFieldError(what=_('SubstitutionMappings'), + field='substitution_filter')) + if any( + key not in { + 'properties', 'capabilities'} for key in value.keys()): + ExceptionCollector.appendException( + UnknownFieldError(what=_('SubstitutionMappings'), + field=key)) + if 'properties' in value.keys(): + if type(tpls_filter[key]['properties']) != list: + ExceptionCollector.appendException( + TypeMismatchError( + what=_( + 'properties of the Node Filter ' + 'definition Keyname'), + type='list')) + if 'capabilities' in value.keys(): + if type(tpls_filter[key]['capabilities']) != list: + ExceptionCollector.appendException( + TypeMismatchError( + what=_( + 'capabilities of the Node Filter ' + 'definition Keyname'), + type='list')) diff --git a/toscaparser/tests/test_topology_template.py b/toscaparser/tests/test_topology_template.py index 38eab7cb..79bb0b41 100644 --- a/toscaparser/tests/test_topology_template.py +++ b/toscaparser/tests/test_topology_template.py @@ -276,6 +276,94 @@ class TopologyTemplateTest(TestCase): exception.ExceptionCollector.assertExceptionMessage( KeyError, errormsg) + def test_substitution_mappings_with_substitution_filter(self): + tpl_path = utils.get_sample_test_path( + "data/topology_template/validate/" + "test_substitution_mappings_section.yaml") + tosca = ToscaTemplate(tpl_path) + filter_name = tosca.tpl['topology_template'][ + 'substitution_mappings']['substitution_filter'].keys() + self.assertEqual('node_filter', list(filter_name)[0]) + n_f = tosca.topology_template.substitution_mappings.sub_mapping_def[ + 'substitution_filter'].keys() + self.assertEqual('node_filter', list(n_f)[0]) + p_v = tosca.topology_template.substitution_mappings.sub_mapping_def[ + 'substitution_filter'].values() + self.assertEqual('property_value', + list(p_v)[0]['properties'][0]['my_property']) + + def test_substitution_mappings_error_substitution_filter(self): + tpl_path = utils.get_sample_test_path( + "data/topology_template/validate/" + "test_substitution_mappings_error_node_filter.yaml") + errormsg = _('SubstitutionMappings contains unknown field ' + '"substitution_filter". Refer to the definition ' + 'to verify valid values.') + err = self.assertRaises( + exception.ValidationError, ToscaTemplate, tpl_path) + self.assertIn(errormsg, str(err)) + + def test_substitution_mappings_error_node_filter(self): + tpl_path = utils.get_sample_test_path( + "data/topology_template/validate/" + "test_substitution_mappings_error_node_filter_key.yaml") + errormsg = _('SubstitutionMappings contains unknown field ' + '"node_filter". Refer to the definition ' + 'to verify valid values.') + err = self.assertRaises( + exception.ValidationError, ToscaTemplate, tpl_path) + self.assertIn(errormsg, str(err)) + + def test_substitution_mappings_error_properties(self): + tpl_path = utils.get_sample_test_path( + "data/topology_template/validate/" + "test_substitution_mappings_error_property_type.yaml") + errormsg = _('properties of the Node Filter definition ' + 'Keyname must be of type "list".') + err = self.assertRaises( + exception.ValidationError, ToscaTemplate, tpl_path) + self.assertIn(errormsg, str(err)) + + def test_substitution_mappings_error_capabilities(self): + tpl_path = utils.get_sample_test_path( + "data/topology_template/validate/" + "test_substitution_mappings_error_capability_type.yaml") + errormsg = _('capabilities of the Node Filter definition ' + 'Keyname must be of type "list".') + err = self.assertRaises( + exception.ValidationError, ToscaTemplate, tpl_path) + self.assertIn(errormsg, str(err)) + + def test_substitution_mappings_with_attributes(self): + tpl_path = utils.get_sample_test_path( + "data/topology_template/validate/" + "test_substitution_mappings_section.yaml") + tosca = ToscaTemplate(tpl_path) + attribute_name = tosca.tpl['topology_template'][ + 'substitution_mappings']['attributes'].keys() + self.assertEqual('receiver_ip', list(attribute_name)[0]) + a_n = tosca.topology_template.substitution_mappings.sub_mapping_def[ + 'attributes'].keys() + self.assertEqual('receiver_ip', list(a_n)[0]) + a_v = tosca.topology_template.substitution_mappings.sub_mapping_def[ + 'attributes'].values() + self.assertEqual('my_property', list(a_v)[0]) + + def test_substitution_mappings_with_interfaces(self): + tpl_path = utils.get_sample_test_path( + "data/topology_template/validate/" + "test_substitution_mappings_section.yaml") + tosca = ToscaTemplate(tpl_path) + interface_name = tosca.tpl['topology_template'][ + 'substitution_mappings']['interfaces'].keys() + self.assertEqual('Standard', list(interface_name)[0]) + i_n = tosca.topology_template.substitution_mappings.sub_mapping_def[ + 'interfaces'].keys() + self.assertEqual('Standard', list(i_n)[0]) + i_v = tosca.topology_template.substitution_mappings.sub_mapping_def[ + 'interfaces'].values() + self.assertEqual('my_start_node.start', list(i_v)[0]['start']) + def test_invalid_type_policies(self): tpl_snippet = ''' policies: diff --git a/toscaparser/tests/test_toscadef.py b/toscaparser/tests/test_toscadef.py index 236f318c..20151938 100644 --- a/toscaparser/tests/test_toscadef.py +++ b/toscaparser/tests/test_toscadef.py @@ -36,6 +36,11 @@ artif_vm_iso_type = ArtifactTypeDef('tosca.artifacts.' 'Deployment.Image.VM.ISO') artif_vm_qcow2_type = ArtifactTypeDef('tosca.artifacts.' 'Deployment.Image.VM.QCOW2') +artif_tpl_type = ArtifactTypeDef('tosca.artifacts.template') +artif_Jinja2_type = ArtifactTypeDef('tosca.artifacts.' + 'template.Jinja2') +artif_Twig_type = ArtifactTypeDef('tosca.artifacts.' + 'template.Twig') policy_root_type = PolicyType('tosca.policies.Root') policy_placement_type = PolicyType('tosca.policies.Placement') policy_scaling_type = PolicyType('tosca.policies.Scaling') @@ -285,6 +290,50 @@ class ToscaDefTest(TestCase): artif_vm_qcow2_type.defs], key=lambda x: str(x))) + self.assertEqual('tosca.artifacts.Root', + artif_tpl_type.parent_type.type) + self.assertEqual({}, artif_tpl_type.parent_artifacts) + self.assertEqual(sorted(['tosca.artifacts.Root', + 'TOSCA base type for template ' + 'type artifacts'], + key=lambda x: str(x)), + sorted([artif_tpl_type. + get_artifact(name) for name in + artif_tpl_type.defs], + key=lambda x: str(x))) + + self.assertEqual('tosca.artifacts.template', + artif_Jinja2_type.parent_type.type) + self.assertEqual({'tosca.artifacts.template': + {'derived_from': 'tosca.artifacts.Root', + 'description': + 'TOSCA base type for template ' + 'type artifacts'}}, + artif_Jinja2_type.parent_artifacts) + self.assertEqual(sorted(['tosca.artifacts.template', + 'Jinja2 template file'], + key=lambda x: str(x)), + sorted([artif_Jinja2_type. + get_artifact(name) for name in + artif_Jinja2_type.defs], + key=lambda x: str(x))) + + self.assertEqual('tosca.artifacts.template', + artif_Twig_type.parent_type.type) + self.assertEqual({'tosca.artifacts.template': + {'derived_from': 'tosca.artifacts.Root', + 'description': + 'TOSCA base type for template ' + 'type artifacts'}}, + artif_Twig_type.parent_artifacts) + self.assertEqual(sorted(['tosca.artifacts.template', + 'Twig template file'], + key=lambda x: str(x)), + sorted([artif_Twig_type. + get_artifact(name) for name in + artif_Twig_type.defs], + key=lambda x: str(x))) + def test_policies(self): self.assertIsNone(policy_root_type.parent_type) self.assertEqual('tosca.policies.Root', diff --git a/toscaparser/tests/test_toscatplvalidation.py b/toscaparser/tests/test_toscatplvalidation.py index cfcb0427..ac7dad65 100644 --- a/toscaparser/tests/test_toscatplvalidation.py +++ b/toscaparser/tests/test_toscatplvalidation.py @@ -867,6 +867,32 @@ tosca-parser/master/toscaparser/tests/data/custom_types/wordpress.yaml TopologyTemplate, tpl, None) self.assertEqual(expectedmessage, err.__str__()) + def test_groups_with_attributes(self): + tpl_snippet = ''' + node_templates: + server: + type: tosca.nodes.Compute + requirements: + - log_endpoint: + capability: log_endpoint + + groups: + webserver_group: + type: tosca.groups.Root + attributes: + cpu_usage: + description: 'Current CPU usage of the node' + value: 75 + members: [ server ] + ''' + tpl = (toscaparser.utils.yamlparser.simple_parse(tpl_snippet)) + groups = TopologyTemplate(tpl, None).groups + self.assertEqual('Current CPU usage of the node', + groups[0].entity_tpl['attributes'] + ['cpu_usage']['description']) + self.assertEqual(75, groups[0].entity_tpl['attributes'] + ['cpu_usage']['value']) + def _custom_types(self): custom_types = {} def_file = utils.get_sample_test_path(