Updated TOSCA1.3 support in tosca-parser

tosca-parser has had some updates to bring it into partial
compliance with the TOSCA Simple Profile for YAML version 1.3.

The implementation for the following updated items has been added:

* 'attributes' was added to the keynames of the Group definition
* 'substitution_filter' was added to the keynames of the Substitution
   mapping
* Added tosca.artifacts.template
* Added tosca.artifacts.template.Jinja2
* Added tosca.artifacts.template.Twig

Change-Id: Iabd748640013e0108d4ac7b4b4fb8a48c412330c

samples/tests/data/topology_template/validate/test_substitution_mappings_error_capability_type.yaml

@@ -0,0 +1,25 @@
+tosca_definitions_version: tosca_simple_yaml_1_3
+
+topology_template:
+  inputs:
+    my_property:
+      type: string
+
+  substitution_mappings:
+    node_type: my.company.MyService
+    substitution_filter:
+      node_filter:
+        properties:
+          - my_property: 'property_value'
+        capabilities:
+          capability_name_or_type_1:
+            properties:
+              - cap_1_property_filter_def_1
+              - cap_1_property_filter_def_2
+
+node_types:
+  my.company.MyService:
+    derived_from: tosca.nodes.Root
+    properties:
+      my_property:
+        type: string

samples/tests/data/topology_template/validate/test_substitution_mappings_error_node_filter.yaml

@@ -0,0 +1,25 @@
+tosca_definitions_version: tosca_simple_yaml_1_3
+
+topology_template:
+  inputs:
+    my_property:
+      type: string
+
+  substitution_mappings:
+    node_type: my.company.MyService
+    substitution_filter:
+      node_filter_test:
+        properties:
+          - my_property: 'property_value'
+        capabilities:
+          - capability_name_or_type_1:
+            properties:
+              - cap_1_property_filter_def_1
+              - cap_1_property_filter_def_2
+
+node_types:
+  my.company.MyService:
+    derived_from: tosca.nodes.Root
+    properties:
+      my_property:
+        type: string

samples/tests/data/topology_template/validate/test_substitution_mappings_error_node_filter_key.yaml

@@ -0,0 +1,25 @@
+tosca_definitions_version: tosca_simple_yaml_1_3
+
+topology_template:
+  inputs:
+    my_property:
+      type: string
+
+  substitution_mappings:
+    node_type: my.company.MyService
+    substitution_filter:
+      node_filter:
+        properties_test:
+          - my_property: 'property_value'
+        capabilities:
+          - capability_name_or_type_1:
+            properties:
+              - cap_1_property_filter_def_1
+              - cap_1_property_filter_def_2
+
+node_types:
+  my.company.MyService:
+    derived_from: tosca.nodes.Root
+    properties:
+      my_property:
+        type: string

samples/tests/data/topology_template/validate/test_substitution_mappings_error_property_type.yaml

@@ -0,0 +1,25 @@
+tosca_definitions_version: tosca_simple_yaml_1_3
+
+topology_template:
+  inputs:
+    my_property:
+      type: string
+
+  substitution_mappings:
+    node_type: my.company.MyService
+    substitution_filter:
+      node_filter:
+        properties:
+          my_property: 'property_value'
+        capabilities:
+          - capability_name_or_type_1:
+            properties:
+              - cap_1_property_filter_def_1
+              - cap_1_property_filter_def_2
+
+node_types:
+  my.company.MyService:
+    derived_from: tosca.nodes.Root
+    properties:
+      my_property:
+        type: string

samples/tests/data/topology_template/validate/test_substitution_mappings_section.yaml

@@ -0,0 +1,51 @@
+tosca_definitions_version: tosca_simple_yaml_1_3
+
+topology_template:
+  inputs:
+    my_property:
+      type: string
+
+  substitution_mappings:
+    node_type: my.company.MyService
+    interfaces:
+      Standard:
+        start: my_start_node.start
+    attributes:
+      receiver_ip: my_property
+    substitution_filter:
+      node_filter:
+        properties:
+          - my_property: 'property_value'
+        capabilities:
+          - capability_name_or_type_1:
+            properties:
+              - cap_1_property_filter_def_1
+              - cap_1_property_filter_def_2
+
+  node_templates:
+    my_start_node:
+      type: my.company.MyServiceCompute
+      properties:
+        my_property: { get_input: my_property }
+      interfaces:
+        Standard:
+          start:
+            implementation: scripts/start.sh
+            inputs:
+              my_property: { get_property: [ SELF, my_property ] }
+
+node_types:
+  my.company.MyService:
+    derived_from: tosca.nodes.Root
+    properties:
+      my_property:
+        type: string
+    attributes:
+      receiver_ip:
+        type: string
+
+  my.company.MyServiceCompute:
+    derived_from: tosca.nodes.Compute
+    properties:
+      my_property:
+        type: string

toscaparser/elements/TOSCA_definition.yaml

@@ -12,10 +12,9 @@
 
 ##########################################################################
 # The content of this file reflects TOSCA Simple Profile in YAML version
-# 1.0.0. It describes the definition for TOSCA types including Node Type,
+# 1.0.3. It describes the definition for TOSCA types including Node Type,
 # Relationship Type, Capability Type and Interfaces.
 ##########################################################################
-tosca_definitions_version: tosca_simple_yaml_1_0
 
 ##########################################################################
 # Node Type.
@@ -918,6 +917,21 @@ artifact_types:
     mime_type: application/octet-stream
     file_ext: [ qcow2 ]
 
+  # Added in TOSCA1.3
+  tosca.artifacts.template:
+    derived_from: tosca.artifacts.Root
+    description: TOSCA base type for template type artifacts
+
+  # Added in TOSCA1.3
+  tosca.artifacts.template.Jinja2:
+    derived_from: tosca.artifacts.template
+    description: Jinja2 template file
+
+  # Added in TOSCA1.3
+  tosca.artifacts.template.Twig:
+    derived_from: tosca.artifacts.template
+    description: Twig template file
+
 ##########################################################################
  # Policy Type.
  # TOSCA Policy Types represent logical grouping of TOSCA nodes that have

toscaparser/elements/entity_type.py

@@ -37,7 +37,7 @@ class EntityType(object):
     '''TOSCA definition file.'''
     TOSCA_DEF_FILE = os.path.join(
         os.path.dirname(os.path.abspath(__file__)),
-        "TOSCA_definition_1_0.yaml")
+        "TOSCA_definition.yaml")
 
     loader = toscaparser.utils.yamlparser.load_yaml
 

toscaparser/elements/grouptype.py

@@ -23,9 +23,9 @@ class GroupType(StatefulEntityType):
     # In TOSCA1.3, the section name 'interfaces' is not defined for group type.
     # It will be ignored if the 'interfaces' is defined as TOSCA1.3.
     SECTIONS = (DERIVED_FROM, VERSION, METADATA, DESCRIPTION, PROPERTIES,
-                MEMBERS, INTERFACES) = \
+                MEMBERS, INTERFACES, ATTRIBUTES) = \
                ("derived_from", "version", "metadata", "description",
-                "properties", "members", "interfaces")
+                "properties", "members", "interfaces", "attributes")
 
     def __init__(self, grouptype, custom_def=None):
         super(GroupType, self).__init__(grouptype, self.GROUP_PREFIX,

toscaparser/entity_template.py

@@ -77,6 +77,7 @@ class EntityTemplate(object):
         self._interfaces = None
         self._requirements = None
         self._capabilities = None
+        self._attributes = None
 
     @property
     def type(self):
@@ -96,6 +97,14 @@ class EntityTemplate(object):
                 self.entity_tpl) or []
         return self._requirements
 
+    @property
+    def attributes(self):
+        if self._attributes is None:
+            self._attributes = self.type_definition.get_value(
+                self.ATTRIBUTES,
+                self.entity_tpl) or []
+        return self._attributes
+
     def get_properties_objects(self):
         '''Return properties objects for this template.'''
         if self._properties is None:

toscaparser/groups.py

@@ -17,9 +17,10 @@ from toscaparser.utils import validateutils
 
 # In TOSCA1.3, the section name 'interfaces' is not defined for groups.
 # It will be ignored if the 'interfaces' is defined as TOSCA1.3.
-SECTIONS = (TYPE, METADATA, DESCRIPTION, PROPERTIES, MEMBERS, INTERFACES) = \
+SECTIONS = (TYPE, METADATA, DESCRIPTION, PROPERTIES, MEMBERS, INTERFACES,
+            ATTRIBUTES) = \
            ('type', 'metadata', 'description',
-            'properties', 'members', 'interfaces')
+            'properties', 'members', 'interfaces', 'attributes')
 
 
 class Group(EntityTemplate):

toscaparser/substitution_mappings.py

@@ -17,6 +17,7 @@ from toscaparser.common.exception import InvalidNodeTypeError
 from toscaparser.common.exception import MissingDefaultValueError
 from toscaparser.common.exception import MissingRequiredFieldError
 from toscaparser.common.exception import MissingRequiredInputError
+from toscaparser.common.exception import TypeMismatchError
 from toscaparser.common.exception import UnknownFieldError
 from toscaparser.common.exception import UnknownOutputError
 from toscaparser.elements.nodetype import NodeType
@@ -32,8 +33,10 @@ class SubstitutionMappings(object):
     implementation of a Node type.
     '''
 
-    SECTIONS = (NODE_TYPE, REQUIREMENTS, CAPABILITIES, PROPERTIES) = \
-               ('node_type', 'requirements', 'capabilities', 'properties')
+    SECTIONS = (NODE_TYPE, REQUIREMENTS, CAPABILITIES, PROPERTIES,
+                ATTRIBUTES, INTERFACES, SUBSTITUTION_FILTER) = \
+               ('node_type', 'requirements', 'capabilities', 'properties',
+                'attributes', 'interfaces', 'substitution_filter')
 
     OPTIONAL_OUTPUTS = ['tosca_id', 'tosca_name', 'state']
 
@@ -77,6 +80,18 @@ class SubstitutionMappings(object):
     def properties(self):
         return self.sub_mapping_def.get(self.PROPERTIES)
 
+    @property
+    def attributes(self):
+        return self.sub_mapping_def.get(self.ATTRIBUTES)
+
+    @property
+    def interfaces(self):
+        return self.sub_mapping_def.get(self.INTERFACES)
+
+    @property
+    def substitution_filter(self):
+        return self.sub_mapping_def.get(self.SUBSTITUTION_FILTER)
+
     @property
     def node_definition(self):
         return NodeType(self.node_type, self.custom_defs)
@@ -92,6 +107,9 @@ class SubstitutionMappings(object):
         self._validate_requirements()
         self._validate_properties()
         self._validate_outputs()
+        self._validate_attributes()
+        self._validate_interfaces()
+        self._validate_substitution_filter()
 
     def _validate_keys(self):
         """validate the keys of substitution mappings."""
@@ -234,3 +252,65 @@ class SubstitutionMappings(object):
                         where=_('SubstitutionMappings with node_type ')
                         + self.node_type,
                         output_name=output.name))
+
+    def _validate_attributes(self):
+        """validate the attributes of substitution mappings."""
+
+        # The attributes must be in node template wchich be mapped.
+        tpls_attributes = self.sub_mapping_def.get(self.ATTRIBUTES)
+        node_attributes = self.sub_mapped_node_template.attributes \
+            if self.sub_mapped_node_template else None
+        for req in node_attributes if node_attributes else []:
+            if (tpls_attributes and
+                    req not in list(tpls_attributes.keys())):
+                pass
+                # ExceptionCollector.appendException(
+                #    UnknownFieldError(what='SubstitutionMappings',
+                #                      field=req))
+
+    def _validate_interfaces(self):
+        """validate the interfaces of substitution mappings."""
+
+        # The interfaces must be in node template wchich be mapped.
+        tpls_interfaces = self.sub_mapping_def.get(self.INTERFACES)
+        node_interfaces = self.sub_mapped_node_template.interfaces \
+            if self.sub_mapped_node_template else None
+        for req in node_interfaces if node_interfaces else []:
+            if (tpls_interfaces and
+                    req not in list(tpls_interfaces.keys())):
+                pass
+                # ExceptionCollector.appendException(
+                #    UnknownFieldError(what='SubstitutionMappings',
+                #                      field=req))
+
+    def _validate_substitution_filter(self):
+        tpls_filter = self.sub_mapping_def.get(
+            self.SUBSTITUTION_FILTER)
+        if tpls_filter:
+            for key, value in tpls_filter.items():
+                if key != 'node_filter':
+                    ExceptionCollector.appendException(
+                        UnknownFieldError(what=_('SubstitutionMappings'),
+                                          field='substitution_filter'))
+                if any(
+                    key not in {
+                        'properties', 'capabilities'} for key in value.keys()):
+                    ExceptionCollector.appendException(
+                        UnknownFieldError(what=_('SubstitutionMappings'),
+                                          field=key))
+                if 'properties' in value.keys():
+                    if type(tpls_filter[key]['properties']) != list:
+                        ExceptionCollector.appendException(
+                            TypeMismatchError(
+                                what=_(
+                                    'properties of the Node Filter '
+                                    'definition Keyname'),
+                                type='list'))
+                if 'capabilities' in value.keys():
+                    if type(tpls_filter[key]['capabilities']) != list:
+                        ExceptionCollector.appendException(
+                            TypeMismatchError(
+                                what=_(
+                                    'capabilities of the Node Filter '
+                                    'definition Keyname'),
+                                type='list'))

toscaparser/tests/test_topology_template.py

@@ -276,6 +276,94 @@ class TopologyTemplateTest(TestCase):
         exception.ExceptionCollector.assertExceptionMessage(
             KeyError, errormsg)
 
+    def test_substitution_mappings_with_substitution_filter(self):
+        tpl_path = utils.get_sample_test_path(
+            "data/topology_template/validate/"
+            "test_substitution_mappings_section.yaml")
+        tosca = ToscaTemplate(tpl_path)
+        filter_name = tosca.tpl['topology_template'][
+            'substitution_mappings']['substitution_filter'].keys()
+        self.assertEqual('node_filter', list(filter_name)[0])
+        n_f = tosca.topology_template.substitution_mappings.sub_mapping_def[
+            'substitution_filter'].keys()
+        self.assertEqual('node_filter', list(n_f)[0])
+        p_v = tosca.topology_template.substitution_mappings.sub_mapping_def[
+            'substitution_filter'].values()
+        self.assertEqual('property_value',
+                         list(p_v)[0]['properties'][0]['my_property'])
+
+    def test_substitution_mappings_error_substitution_filter(self):
+        tpl_path = utils.get_sample_test_path(
+            "data/topology_template/validate/"
+            "test_substitution_mappings_error_node_filter.yaml")
+        errormsg = _('SubstitutionMappings contains unknown field '
+                     '"substitution_filter". Refer to the definition '
+                     'to verify valid values.')
+        err = self.assertRaises(
+            exception.ValidationError, ToscaTemplate, tpl_path)
+        self.assertIn(errormsg, str(err))
+
+    def test_substitution_mappings_error_node_filter(self):
+        tpl_path = utils.get_sample_test_path(
+            "data/topology_template/validate/"
+            "test_substitution_mappings_error_node_filter_key.yaml")
+        errormsg = _('SubstitutionMappings contains unknown field '
+                     '"node_filter". Refer to the definition '
+                     'to verify valid values.')
+        err = self.assertRaises(
+            exception.ValidationError, ToscaTemplate, tpl_path)
+        self.assertIn(errormsg, str(err))
+
+    def test_substitution_mappings_error_properties(self):
+        tpl_path = utils.get_sample_test_path(
+            "data/topology_template/validate/"
+            "test_substitution_mappings_error_property_type.yaml")
+        errormsg = _('properties of the Node Filter definition '
+                     'Keyname must be of type "list".')
+        err = self.assertRaises(
+            exception.ValidationError, ToscaTemplate, tpl_path)
+        self.assertIn(errormsg, str(err))
+
+    def test_substitution_mappings_error_capabilities(self):
+        tpl_path = utils.get_sample_test_path(
+            "data/topology_template/validate/"
+            "test_substitution_mappings_error_capability_type.yaml")
+        errormsg = _('capabilities of the Node Filter definition '
+                     'Keyname must be of type "list".')
+        err = self.assertRaises(
+            exception.ValidationError, ToscaTemplate, tpl_path)
+        self.assertIn(errormsg, str(err))
+
+    def test_substitution_mappings_with_attributes(self):
+        tpl_path = utils.get_sample_test_path(
+            "data/topology_template/validate/"
+            "test_substitution_mappings_section.yaml")
+        tosca = ToscaTemplate(tpl_path)
+        attribute_name = tosca.tpl['topology_template'][
+            'substitution_mappings']['attributes'].keys()
+        self.assertEqual('receiver_ip', list(attribute_name)[0])
+        a_n = tosca.topology_template.substitution_mappings.sub_mapping_def[
+            'attributes'].keys()
+        self.assertEqual('receiver_ip', list(a_n)[0])
+        a_v = tosca.topology_template.substitution_mappings.sub_mapping_def[
+            'attributes'].values()
+        self.assertEqual('my_property', list(a_v)[0])
+
+    def test_substitution_mappings_with_interfaces(self):
+        tpl_path = utils.get_sample_test_path(
+            "data/topology_template/validate/"
+            "test_substitution_mappings_section.yaml")
+        tosca = ToscaTemplate(tpl_path)
+        interface_name = tosca.tpl['topology_template'][
+            'substitution_mappings']['interfaces'].keys()
+        self.assertEqual('Standard', list(interface_name)[0])
+        i_n = tosca.topology_template.substitution_mappings.sub_mapping_def[
+            'interfaces'].keys()
+        self.assertEqual('Standard', list(i_n)[0])
+        i_v = tosca.topology_template.substitution_mappings.sub_mapping_def[
+            'interfaces'].values()
+        self.assertEqual('my_start_node.start', list(i_v)[0]['start'])
+
     def test_invalid_type_policies(self):
         tpl_snippet = '''
         policies:

toscaparser/tests/test_toscadef.py

@@ -36,6 +36,11 @@ artif_vm_iso_type = ArtifactTypeDef('tosca.artifacts.'
                                     'Deployment.Image.VM.ISO')
 artif_vm_qcow2_type = ArtifactTypeDef('tosca.artifacts.'
                                       'Deployment.Image.VM.QCOW2')
+artif_tpl_type = ArtifactTypeDef('tosca.artifacts.template')
+artif_Jinja2_type = ArtifactTypeDef('tosca.artifacts.'
+                                    'template.Jinja2')
+artif_Twig_type = ArtifactTypeDef('tosca.artifacts.'
+                                  'template.Twig')
 policy_root_type = PolicyType('tosca.policies.Root')
 policy_placement_type = PolicyType('tosca.policies.Placement')
 policy_scaling_type = PolicyType('tosca.policies.Scaling')
@@ -285,6 +290,50 @@ class ToscaDefTest(TestCase):
                                 artif_vm_qcow2_type.defs],
                                 key=lambda x: str(x)))
 
+        self.assertEqual('tosca.artifacts.Root',
+                         artif_tpl_type.parent_type.type)
+        self.assertEqual({}, artif_tpl_type.parent_artifacts)
+        self.assertEqual(sorted(['tosca.artifacts.Root',
+                                 'TOSCA base type for template '
+                                 'type artifacts'],
+                                key=lambda x: str(x)),
+                         sorted([artif_tpl_type.
+                                get_artifact(name) for name in
+                                artif_tpl_type.defs],
+                                key=lambda x: str(x)))
+
+        self.assertEqual('tosca.artifacts.template',
+                         artif_Jinja2_type.parent_type.type)
+        self.assertEqual({'tosca.artifacts.template':
+                          {'derived_from': 'tosca.artifacts.Root',
+                           'description':
+                           'TOSCA base type for template '
+                           'type artifacts'}},
+                         artif_Jinja2_type.parent_artifacts)
+        self.assertEqual(sorted(['tosca.artifacts.template',
+                                 'Jinja2 template file'],
+                                key=lambda x: str(x)),
+                         sorted([artif_Jinja2_type.
+                                get_artifact(name) for name in
+                                artif_Jinja2_type.defs],
+                                key=lambda x: str(x)))
+
+        self.assertEqual('tosca.artifacts.template',
+                         artif_Twig_type.parent_type.type)
+        self.assertEqual({'tosca.artifacts.template':
+                          {'derived_from': 'tosca.artifacts.Root',
+                           'description':
+                           'TOSCA base type for template '
+                           'type artifacts'}},
+                         artif_Twig_type.parent_artifacts)
+        self.assertEqual(sorted(['tosca.artifacts.template',
+                                 'Twig template file'],
+                                key=lambda x: str(x)),
+                         sorted([artif_Twig_type.
+                                get_artifact(name) for name in
+                                artif_Twig_type.defs],
+                                key=lambda x: str(x)))
+
     def test_policies(self):
         self.assertIsNone(policy_root_type.parent_type)
         self.assertEqual('tosca.policies.Root',

toscaparser/tests/test_toscatplvalidation.py

@@ -867,6 +867,32 @@ tosca-parser/master/toscaparser/tests/data/custom_types/wordpress.yaml
                                 TopologyTemplate, tpl, None)
         self.assertEqual(expectedmessage, err.__str__())
 
+    def test_groups_with_attributes(self):
+        tpl_snippet = '''
+        node_templates:
+          server:
+            type: tosca.nodes.Compute
+            requirements:
+              - log_endpoint:
+                  capability: log_endpoint
+
+        groups:
+          webserver_group:
+            type: tosca.groups.Root
+            attributes:
+              cpu_usage:
+                description: 'Current CPU usage of the node'
+                value: 75
+              members: [ server ]
+        '''
+        tpl = (toscaparser.utils.yamlparser.simple_parse(tpl_snippet))
+        groups = TopologyTemplate(tpl, None).groups
+        self.assertEqual('Current CPU usage of the node',
+                         groups[0].entity_tpl['attributes']
+                         ['cpu_usage']['description'])
+        self.assertEqual(75, groups[0].entity_tpl['attributes']
+                         ['cpu_usage']['value'])
+
     def _custom_types(self):
         custom_types = {}
         def_file = utils.get_sample_test_path(