Support for updating tosca-parser with TOSCA1.3

tosca-parser has had some updates to bring it into partial
compliance with the TOSCA Simple Profile for YAML version 1.3.

The implementation for the following updated items has been added:

The keyname of the Trigger definition was changed
'key_schema' was added to the keynames of the Property definition
'key_schema' was added to the keynames of the Parameter definition
The grammar for 'operation_definitions' in Interface Type changed.
'notifications' was added to the keynames of the Interface definition

Change-Id: I59f2c4b3f565a290e0f22d3648c9a2618dc1bc4e
Signed-off-by: Yasufumi Ogawa <yasufum.o@gmail.com>
Co-Authored-By: Kaori Mitani <mitani.kaori@fujitsu.com>

samples/tests/data/interfaces/test_custom_interface_notifications.yaml

@@ -0,0 +1,32 @@
+tosca_definitions_version: tosca_simple_yaml_1_3
+
+description: >
+  This template contains custom defined interface type
+  and a node type which uses this custom interface
+
+topology_template:
+
+  node_templates:
+    customInterfaceTest:
+      type: tosca.nodes.CustomInterfaceTest
+      interfaces:
+        CustomInterface:
+          operations:
+            CustomOp:
+          notifications:
+            CustomNo:
+
+interface_types:
+  tosca.interfaces.CustomInterface:
+    derived_from: tosca.interfaces.Root
+    operations:
+      CustomOp:
+    notifications:
+      CustomNo:
+
+node_types:
+  tosca.nodes.CustomInterfaceTest:
+    derived_from: tosca.nodes.WebApplication
+    interfaces:
+      CustomInterface:
+        type: tosca.interfaces.CustomInterface

samples/tests/data/interfaces/test_custom_interface_operations.yaml

@@ -0,0 +1,29 @@
+tosca_definitions_version: tosca_simple_yaml_1_3
+
+description: >
+  This template contains custom defined interface type
+  and a node type which uses this custom interface
+
+topology_template:
+
+  node_templates:
+    customInterfaceTest:
+      type: tosca.nodes.CustomInterfaceTest
+      interfaces:
+        CustomInterface:
+          operations:
+            CustomOp: # operation from interface_type with additional outputs
+              outputs:
+
+interface_types:
+  tosca.interfaces.CustomInterface:
+    derived_from: tosca.interfaces.Root
+    operations:
+      CustomOp:
+
+node_types:
+  tosca.nodes.CustomInterfaceTest:
+    derived_from: tosca.nodes.WebApplication
+    interfaces:
+      CustomInterface:
+        type: tosca.interfaces.CustomInterface

samples/tests/data/relationship/test_custom_relationship_key_schema.yaml

@@ -0,0 +1,21 @@
+tosca_definitions_version: tosca_simple_yaml_1_3
+
+description: >
+  This template contains custom defined interface type
+  and a node type which uses this custom interface
+
+topology_template:
+  relationship_templates:
+    customRelationshipTest:
+      type: tosca.nodes.CustomRelationshipTest
+      properties:
+        property_test:
+          'test_key': 'test_value'
+
+relationship_types:
+  tosca.nodes.CustomRelationshipTest:
+    properties:
+      property_test:
+        type: map
+        key_schema:
+          type: string

toscaparser/dataentity.py

@@ -98,7 +98,8 @@ class DataEntity(object):
                 # check if field value meets type defined
                 DataEntity.validate_datatype(prop_schema.type, value,
                                              prop_schema.entry_schema,
-                                             self.custom_def)
+                                             self.custom_def, None,
+                                             prop_schema.key_schema)
                 # check if field value meets constraints defined
                 if prop_schema.constraints:
                     for constraint in prop_schema.constraints:
@@ -115,8 +116,8 @@ class DataEntity(object):
             return self.schema[name].schema
 
     @staticmethod
-    def validate_datatype(type, value, entry_schema=None, custom_def=None,
-                          prop_name=None):
+    def validate_datatype(type, value, entry_schema=None,
+                          custom_def=None, prop_name=None, key_schema=None):
         '''Validate value with given type.
 
         If type is list or map, validate its entry by entry_schema(if defined)
@@ -155,6 +156,8 @@ class DataEntity(object):
             return validateutils.TOSCAVersionProperty(value).get_version()
         elif type == Schema.MAP:
             validateutils.validate_map(value)
+            if key_schema:
+                DataEntity.validate_key(value, key_schema, custom_def)
             if entry_schema:
                 DataEntity.validate_entry(value, entry_schema, custom_def)
             return value
@@ -174,8 +177,26 @@ class DataEntity(object):
         if isinstance(value, dict):
             valuelist = list(value.values())
         for v in valuelist:
-            DataEntity.validate_datatype(schema.type, v, schema.entry_schema,
-                                         custom_def)
+            DataEntity.validate_datatype(schema.type, v,
+                                         schema.entry_schema,
+                                         custom_def, None,
+                                         schema.key_schema)
+            if schema.constraints:
+                for constraint in schema.constraints:
+                    constraint.validate(v)
+        return value
+
+    def validate_key(value, key_schema, custom_def=None):
+        '''Validate keys for map'''
+        schema = Schema(None, key_schema)
+        valuelist = value
+        if isinstance(value, dict):
+            valuelist = list(value.keys())
+        for v in valuelist:
+            DataEntity.validate_datatype(schema.type, v,
+                                         schema.entry_schema,
+                                         custom_def, None,
+                                         schema.key_schema)
             if schema.constraints:
                 for constraint in schema.constraints:
                     constraint.validate(v)

toscaparser/elements/constraints.py

@@ -27,10 +27,10 @@ class Schema(collections.abc.Mapping):
 
     KEYS = (
         TYPE, REQUIRED, DESCRIPTION,
-        DEFAULT, CONSTRAINTS, ENTRYSCHEMA, STATUS
+        DEFAULT, CONSTRAINTS, KEYSCHEMA, ENTRYSCHEMA, STATUS
     ) = (
         'type', 'required', 'description',
-        'default', 'constraints', 'entry_schema', 'status'
+        'default', 'constraints', 'key_schema', 'entry_schema', 'status'
     )
 
     PROPERTY_TYPES = (
@@ -100,6 +100,10 @@ class Schema(collections.abc.Mapping):
                                          for cschema in constraint_schemata]
         return self.constraints_list
 
+    @property
+    def key_schema(self):
+        return self.schema.get(self.KEYSCHEMA)
+
     @property
     def entry_schema(self):
         return self.schema.get(self.ENTRYSCHEMA)

toscaparser/elements/grouptype.py

@@ -20,6 +20,8 @@ from toscaparser.elements.statefulentitytype import StatefulEntityType
 class GroupType(StatefulEntityType):
     '''TOSCA built-in group type.'''
 
+    # In TOSCA1.3, the section name 'interfaces' is not defined for group type.
+    # It will be ignored if the 'interfaces' is defined as TOSCA1.3.
     SECTIONS = (DERIVED_FROM, VERSION, METADATA, DESCRIPTION, PROPERTIES,
                 MEMBERS, INTERFACES) = \
                ("derived_from", "version", "metadata", "description",

toscaparser/entity_template.py

@@ -310,6 +310,24 @@ class EntityTemplate(object):
                                                              self.entity_tpl)
         if type_interfaces:
             for interface_type, value in type_interfaces.items():
+                # If there is 'notifications' as a key name in the Interface
+                # definition, it will be determined that it is TOSCA1.3 and
+                # will be processed.
+                if 'notifications' in value:
+                    value_notifications = value['notifications']
+                    for no, no_def in value_notifications.items():
+                        iface = InterfacesDef(self.type_definition,
+                                              interfacename=interface_type,
+                                              node_template=self,
+                                              name=no,
+                                              value=no_def)
+                        interfaces.append(iface)
+                # If there is 'operations' as a key name in the Interface
+                # definition, it will be determined that it is TOSCA1.3 and
+                # will be processed. As a result of this process, the
+                # operation_name named 'operations' cannot be set in TOSCA1.2.
+                if 'operations' in value:
+                    value = value['operations']
                 for op, op_def in value.items():
                     iface = InterfacesDef(self.type_definition,
                                           interfacename=interface_type,

toscaparser/groups.py

@@ -15,6 +15,8 @@ from toscaparser.common.exception import UnknownFieldError
 from toscaparser.entity_template import EntityTemplate
 from toscaparser.utils import validateutils
 
+# In TOSCA1.3, the section name 'interfaces' is not defined for groups.
+# It will be ignored if the 'interfaces' is defined as TOSCA1.3.
 SECTIONS = (TYPE, METADATA, DESCRIPTION, PROPERTIES, MEMBERS, INTERFACES) = \
            ('type', 'metadata', 'description',
             'properties', 'members', 'interfaces')

toscaparser/nodetemplate.py

@@ -254,6 +254,16 @@ class NodeTemplate(EntityTemplate):
                                                 self.entity_tpl)
         if ifaces:
             for name, value in ifaces.items():
+                # Corresponding to 'notifications' Keyname addition in TOSCA1.3
+                if 'notifications' in value:
+                    value_notifications = value['notifications']
+                    self._common_validate_field(
+                        value_notifications,
+                        self._collect_custom_iface_notifications(name),
+                        'interfaces')
+                # Corresponding to 'operations' Keyname addition in TOSCA1.3
+                if 'operations' in value:
+                    value = value['operations']
                 if name in (LIFECYCLE, LIFECYCLE_SHORTNAME):
                     self._common_validate_field(
                         value, InterfacesDef.
@@ -286,7 +296,14 @@ class NodeTemplate(EntityTemplate):
         if 'type' in nodetype_iface_def:
             iface_type = nodetype_iface_def['type']
             if iface_type in self.type_definition.custom_def:
-                iface_type_def = self.type_definition.custom_def[iface_type]
+                # Corresponding to 'operations' Keyname addition in TOSCA1.3
+                operations = self.type_definition.custom_def[iface_type].get(
+                    'operations', {})
+                if operations:
+                    iface_type_def = operations
+                else:
+                    iface_type_def = self.type_definition.custom_def[
+                        iface_type]
             else:
                 iface_type_def = self.type_definition.TOSCA_DEF[iface_type]
             allowed_operations.extend(iface_type_def.keys())
@@ -294,6 +311,22 @@ class NodeTemplate(EntityTemplate):
                               op not in INTERFACE_DEF_RESERVED_WORDS]
         return allowed_operations
 
+    def _collect_custom_iface_notifications(self, name):
+        allowed_notifications = []
+        nodetype_iface_def = self.type_definition.interfaces[name]
+        allowed_notifications.extend(nodetype_iface_def.keys())
+        if 'type' in nodetype_iface_def:
+            iface_type = nodetype_iface_def['type']
+            if iface_type in self.type_definition.custom_def:
+                iface_type_def = self.type_definition.custom_def[
+                    iface_type]['notifications']
+            else:
+                iface_type_def = self.type_definition.TOSCA_DEF[iface_type]
+            allowed_notifications.extend(iface_type_def.keys())
+        allowed_notifications = [no for no in allowed_notifications if
+                                 no not in INTERFACE_DEF_RESERVED_WORDS]
+        return allowed_notifications
+
     def _validate_fields(self, nodetemplate):
         for name in nodetemplate.keys():
             if name not in self.SECTIONS and name not in self.SPECIAL_SECTIONS:

toscaparser/parameters.py

@@ -28,9 +28,10 @@ log = logging.getLogger('tosca')
 class Input(object):
 
     INPUTFIELD = (TYPE, DESCRIPTION, DEFAULT, CONSTRAINTS, REQUIRED, STATUS,
-                  ENTRY_SCHEMA) = ('type', 'description', 'default',
-                                   'constraints', 'required', 'status',
-                                   'entry_schema')
+                  KEY_SCHEMA, ENTRY_SCHEMA) = ('type', 'description',
+                                               'default', 'constraints',
+                                               'required', 'status',
+                                               'key_schema', 'entry_schema')
 
     def __init__(self, name, schema_dict):
         self.name = name

toscaparser/properties.py

@@ -56,6 +56,10 @@ class Property(object):
     def constraints(self):
         return self.schema.constraints
 
+    @property
+    def key_schema(self):
+        return self.schema.key_schema
+
     @property
     def entry_schema(self):
         return self.schema.entry_schema
@@ -68,7 +72,8 @@ class Property(object):
             self.value = DataEntity.validate_datatype(self.type, self.value,
                                                       self.entry_schema,
                                                       self.custom_def,
-                                                      self.name)
+                                                      self.name,
+                                                      self.key_schema)
             self._validate_constraints()
 
     def _validate_constraints(self):

toscaparser/tests/test_toscatplvalidation.py

@@ -56,6 +56,42 @@ class ToscaTemplateValidationTest(TestCase):
               'contains unknown field "CustomOp4". '
               'Refer to the definition to verify valid values.'))
 
+    # Since 'operations' Keyname was defined in TOSCA1.3,
+    # test whether operation_definitions defined
+    # under 'operations' can be used.
+    def test_custom_interface_operations(self):
+        tpl_path = TestCase.test_sample(
+            "data/interfaces/test_custom_interface_operations.yaml")
+        tosca = ToscaTemplate(tpl_path)
+        op_names = tosca.tpl['interface_types'][
+            'tosca.interfaces.CustomInterface']['operations'].keys()
+        self.assertEqual('CustomOp', list(op_names)[0])
+        op_names = tosca.topology_template.custom_defs[
+            'tosca.interfaces.CustomInterface']['operations'].keys()
+        self.assertEqual('CustomOp', list(op_names)[0])
+        op_names = tosca.topology_template.nodetemplates[0].templates[
+            'customInterfaceTest']['interfaces']['CustomInterface'][
+                'operations'].keys()
+        self.assertEqual('CustomOp', list(op_names)[0])
+
+    # Since 'notifications' Keyname was defined in TOSCA1.3,
+    # test whether notification_definitions defined
+    # under 'notifications' can be used.
+    def test_custom_interface_notifications(self):
+        tpl_path = TestCase.test_sample(
+            "data/interfaces/test_custom_interface_notifications.yaml")
+        tosca = ToscaTemplate(tpl_path)
+        no_names = tosca.tpl['interface_types'][
+            'tosca.interfaces.CustomInterface']['notifications'].keys()
+        self.assertEqual('CustomNo', list(no_names)[0])
+        no_names = tosca.topology_template.custom_defs[
+            'tosca.interfaces.CustomInterface']['notifications'].keys()
+        self.assertEqual('CustomNo', list(no_names)[0])
+        no_names = tosca.topology_template.nodetemplates[0].templates[
+            'customInterfaceTest']['interfaces']['CustomInterface'][
+                'notifications'].keys()
+        self.assertEqual('CustomNo', list(no_names)[0])
+
     def test_first_level_sections(self):
         tpl_path = TestCase.test_sample(
             "data/test_tosca_top_level_error1.yaml")
@@ -273,6 +309,15 @@ class ToscaTemplateValidationTest(TestCase):
               type: string
             default: []
         '''
+        tpl_snippet4 = '''
+        inputs:
+          some_list:
+            type: list
+            description: List of items
+            key_schema:
+              type: string
+            default: []
+        '''
         inputs1 = (toscaparser.utils.yamlparser.
                    simple_parse(tpl_snippet1)['inputs'])
         name1, attrs1 = list(inputs1.items())[0]
@@ -289,6 +334,9 @@ class ToscaTemplateValidationTest(TestCase):
         input2 = Input(name2, attrs2)
         self.assertTrue(input2.required)
         toscaparser.utils.yamlparser.simple_parse(tpl_snippet3)['inputs']
+        input4 = (toscaparser.utils.yamlparser.
+                  simple_parse(tpl_snippet4)['inputs'])
+        self.assertEqual('string', input4['some_list']['key_schema']['type'])
 
     def _imports_content_test(self, tpl_snippet, path, custom_type_def):
         imports = (toscaparser.utils.yamlparser.
@@ -1494,6 +1542,23 @@ tosca-parser/master/toscaparser/tests/data/custom_types/wordpress.yaml
                                 rel_template.validate)
         self.assertEqual(expectedmessage, str(err))
 
+    def test_relationship_template_properties_key_schema(self):
+        tpl_path = TestCase.test_sample(
+            "data/relationship/test_custom_relationship_key_schema.yaml")
+        tosca = ToscaTemplate(tpl_path)
+        p_key_schema_type = tosca.tpl['relationship_types'][
+            'tosca.nodes.CustomRelationshipTest']['properties'][
+                'property_test']['key_schema']['type']
+        self.assertEqual('string', p_key_schema_type)
+        p_key_schema_type = tosca.relationship_templates[0].custom_def[
+            'tosca.nodes.CustomRelationshipTest']['properties'][
+                'property_test']['key_schema']['type']
+        self.assertEqual('string', p_key_schema_type)
+        property_value = (
+            tosca.topology_template.relationship_templates[0].entity_tpl[
+                'properties']['property_test']['test_key'])
+        self.assertEqual('test_value', property_value)
+
     def test_tosca_version_1_3(self):
         tpl_path = TestCase.test_sample(
             "data/test_tosca_version_1_3.yaml")
@@ -1710,6 +1775,39 @@ tosca-parser/master/toscaparser/tests/data/custom_types/wordpress.yaml
         name = list(triggers.keys())[0]
         Triggers(name, triggers[name])
 
+    # Since the keyname 'event_type' has been changed in TOSCA1.3,
+    # test whether the new keyname 'event' can be used.
+    def test_policy_trigger_keyname_event(self):
+        tpl_snippet = '''
+        triggers:
+         - resize_compute:
+             description: trigger
+             event: tosca.events.resource.utilization
+             schedule:
+               start_time: "2015-05-07T07:00:00Z"
+               end_time: "2015-06-07T07:00:00Z"
+             target_filter:
+               node: master-container
+               requirement: host
+               capability: Container
+             condition:
+               constraint: { greater_than: 50 }
+               granularity: 60
+               evaluations: 1
+               aggregation_method : mean
+             action:
+               resize: # Operation name
+                inputs:
+                 strategy: LEAST_USED
+                 implementation: Senlin.webhook()
+        '''
+        triggers = (toscaparser.utils.yamlparser.
+                    simple_parse(tpl_snippet))['triggers'][0]
+        name = list(triggers.keys())[0]
+        trigger = Triggers(name, triggers[name])
+        self.assertEqual('tosca.events.resource.utilization',
+                         trigger.get_event())
+
     def test_policy_trigger_valid_keyname_heat_resources(self):
         tpl_snippet = '''
         triggers:

toscaparser/triggers.py

@@ -18,9 +18,9 @@ from toscaparser.common.exception import UnknownFieldError
 from toscaparser.entity_template import EntityTemplate
 from toscaparser.utils import validateutils
 
-SECTIONS = (DESCRIPTION, EVENT, SCHEDULE, METRIC, METADATA,
+SECTIONS = (DESCRIPTION, EVENT_TYPE, EVENT, SCHEDULE, METRIC, METADATA,
             TARGET_FILTER, CONDITION, ACTION) = \
-           ('description', 'event_type', 'schedule', 'metric',
+           ('description', 'event_type', 'event', 'schedule', 'metric',
             'metadata', 'target_filter', 'condition', 'action')
 CONDITION_KEYNAMES = (CONSTRAINT, GRANULARITY, EVALUATIONS, AGGREGATION_METHOD,
                       THRESHOLD, COMPARISON_OPERATOR, RESOURCE_TYPE, STATE) = \
@@ -45,6 +45,9 @@ class Triggers(EntityTemplate):
         return self.trigger_tpl['description']
 
     def get_event(self):
+        # Corresponding to Keyname changes in TOSCA1.3
+        if self.trigger_tpl.get('event'):
+            return self.trigger_tpl['event']
         return self.trigger_tpl['event_type']
 
     def get_schedule(self):