Add input validation in substitution_mapping class

Add input validation in class of substitution_mapping according to
specification of
http://docs.oasis-open.org/tosca/TOSCA-Simple-Profile-YAML/v1.0/
TOSCA-Simple-Profile-YAML-v1.0.html:
1) The properties of substituted node template which be mapped must be
in the inputs of nested service template which defines substutition
mappings;
2) The inputs of nested service template which are not in properties of
the substituted node template must have default values.
3) If the properties of node_type is required and no default value,
 must provide inputs for them;
4) Property names and the input names must be the same.

This patch is related to bp:https://review.openstack.org/#/c/345492/

Change-Id: Ib928434ec67661e689ac80cca1749b53d17e4ba8
Signed-off-by: shangxdy <shang.xiaodong@zte.com.cn>

toscaparser/common/exception.py

@@ -114,11 +114,21 @@ class UnknownInputError(TOSCAException):
     msg_fmt = _('Unknown input "%(input_name)s".')
 
 
+class MissingRequiredInputError(TOSCAException):
+    msg_fmt = _('%(what)s is missing required input definition '
+                'of input "%(input_name)s".')
+
+
 class MissingRequiredParameterError(TOSCAException):
-    msg_fmt = _('%(what)s is missing required parameter for input: '
+    msg_fmt = _('%(what)s is missing required parameter for input '
                 '"%(input_name)s".')
 
 
+class MissingDefaultValueError(TOSCAException):
+    msg_fmt = _('%(what)s is missing required default value '
+                'of input "%(input_name)s".')
+
+
 class InvalidPropertyValueError(TOSCAException):
     msg_fmt = _('Value of property "%(what)s" is invalid.')
 

toscaparser/substitution_mappings.py

@@ -14,9 +14,12 @@ import logging
 
 from toscaparser.common.exception import ExceptionCollector
 from toscaparser.common.exception import InvalidNodeTypeError
+from toscaparser.common.exception import MissingDefaultValueError
 from toscaparser.common.exception import MissingRequiredFieldError
+from toscaparser.common.exception import MissingRequiredInputError
 from toscaparser.common.exception import UnknownFieldError
-
+from toscaparser.elements.nodetype import NodeType
+from toscaparser.utils.gettextutils import _
 
 log = logging.getLogger('tosca')
 
@@ -47,7 +50,7 @@ class SubstitutionMappings(object):
     @property
     def type(self):
         if self.sub_mapping_def:
-            return self.sub_mapping_def['node_type']
+            return self.sub_mapping_def.get(self.NODE_TYPE)
 
     @classmethod
     def get_node_type(cls, sub_mapping_def):
@@ -66,9 +69,16 @@ class SubstitutionMappings(object):
     def requirements(self):
         return self.sub_mapping_def.get(self.REQUIREMENTS)
 
+    @property
+    def node_definition(self):
+        return NodeType(self.node_type, self.custom_defs)
+
     def _validate(self):
+        # Basic validation
         self._validate_keys()
         self._validate_type()
+
+        # SubstitutionMapping class syntax validation
         self._validate_inputs()
         self._validate_capabilities()
         self._validate_requirements()
@@ -79,7 +89,7 @@ class SubstitutionMappings(object):
         for key in self.sub_mapping_def.keys():
             if key not in self.SECTIONS:
                 ExceptionCollector.appendException(
-                    UnknownFieldError(what='SubstitutionMappings',
+                    UnknownFieldError(what=_('SubstitutionMappings'),
                                       field=key))
 
     def _validate_type(self):
@@ -94,24 +104,59 @@ class SubstitutionMappings(object):
         node_type_def = self.custom_defs.get(node_type)
         if not node_type_def:
             ExceptionCollector.appendException(
-                InvalidNodeTypeError(what=node_type_def))
+                InvalidNodeTypeError(what=node_type))
 
     def _validate_inputs(self):
-        """validate the inputs of substitution mappings."""
+        """validate the inputs of substitution mappings.
 
-        # The inputs in service template which defines substutition mappings
-        # must be in properties of node template which is mapped or provide
-        # defualt value. Currently the input.name is not restrict to be the
-        # same as properte's name in specification, but they should be equal
-        # for current implementation.
-        property_names = list(self.sub_mapped_node_template
-                              .get_properties().keys()
-                              if self.sub_mapped_node_template else [])
-        for input in self.inputs:
-            if input.name not in property_names and input.default is None:
+        The inputs defined by the topology template have to match the
+        properties of the node type or the substituted node. If there are
+        more inputs than the substituted node has properties, default values
+        must be defined for those inputs.
+        """
+
+        all_inputs = set([input.name for input in self.inputs])
+        required_properties = set([p.name for p in
+                                   self.node_definition.
+                                   get_properties_def_objects()
+                                   if p.required and p.default is None])
+        # Must provide inputs for required properties of node type.
+        for property in required_properties:
+            # Check property which is 'required' and has no 'default' value
+            if property not in all_inputs:
                 ExceptionCollector.appendException(
-                    UnknownFieldError(what='SubstitutionMappings',
-                                      field=input.name))
+                    MissingRequiredInputError(
+                        what=_('SubstitutionMappings with node_type ')
+                        + self.node_type,
+                        input_name=property))
+
+        # If the optional properties of node type need to be customized by
+        # substituted node, it also is necessary to define inputs for them,
+        # otherwise they are not mandatory to be defined.
+        customized_parameters = set(self.sub_mapped_node_template
+                                    .get_properties().keys()
+                                    if self.sub_mapped_node_template else [])
+        all_properties = set(self.node_definition.get_properties_def())
+        for parameter in customized_parameters - all_inputs:
+            if parameter in all_properties:
+                ExceptionCollector.appendException(
+                    MissingRequiredInputError(
+                        what=_('SubstitutionMappings with node_type ')
+                        + self.node_type,
+                        input_name=parameter))
+
+        # Additional inputs are not in the properties of node type must
+        # provide default values. Currently the scenario may not happen
+        # because of parameters validation in nodetemplate, here is a
+        # guarantee.
+        for input in self.inputs:
+            if input.name in all_inputs - all_properties \
+               and input.default is None:
+                ExceptionCollector.appendException(
+                    MissingDefaultValueError(
+                        what=_('SubstitutionMappings with node_type ')
+                        + self.node_type,
+                        input_name=input.name))
 
     def _validate_capabilities(self):
         """validate the capabilities of substitution mappings."""

toscaparser/tests/data/topology_template/validate/queuingsubsystem_invalid_input.yaml

@@ -0,0 +1,76 @@
+tosca_definitions_version: tosca_simple_yaml_1_0
+
+description: >
+  This template is a test template which contains invalid input needed for substitution mappings.
+  The required properties without default value in substituted node template which be mapped must be
+  as inputs of nested service template which defines substutition mappings, and the inputs of nested
+  service template which are not in the properties of the substituted node template must provide
+  default values.
+  This template provides an additional input of server_port1/my_cpus/my_input which are not defined
+  in example.QueuingSubsystem, and the default value are 8080/2/123, all of these are right. But the
+  required property of server_port defined in example.QueuingSubsystem is not appeared in inputs
+  definiton, so will raise excepton of "MissingRequiredInputError".
+
+imports:
+  - ../definitions.yaml
+
+topology_template:
+  description: Template of a database including its hosting stack.
+
+  inputs:
+    server_ip:
+      type: string
+      description: IP address of the message queuing server to receive messages from.
+      default: 127.0.0.1
+    server_port1:
+      type: integer
+      description: Port to be used for receiving messages.
+      default: 8080
+    my_cpus:
+      type: integer
+      description: Number of CPUs for the server.
+      default: 2
+      constraints:
+        - valid_values: [ 1, 2, 4, 8 ]
+    my_input:
+      type: integer
+      description: test for input validation.
+      default: 123
+
+  substitution_mappings:
+    node_type: example.QueuingSubsystem
+
+  node_templates:
+    tran_app:
+      type: example.QueuingSubsystem
+      properties:
+        server_ip: { get_input: server_ip }
+        server_port: { get_input: server_port1 }
+      requirements:
+        - host:
+            node: server
+
+    server:
+      type: tosca.nodes.Compute
+      capabilities:
+        host:
+          properties:
+            disk_size: 10 GB
+            num_cpus: { get_input: my_cpus }
+            mem_size: 4096 MB
+        os:
+          properties:
+            architecture: x86_64
+            type: Linux
+            distribution: Ubuntu
+            version: 14.04
+
+  outputs:
+    receiver_ip:
+      description: private IP address of the message receiver application
+      value: { get_attribute: [ server, private_address ] }
+
+  groups:
+    tran_server_group:
+      members: [ tran_app, server ]
+      type: tosca.groups.Root

toscaparser/tests/data/topology_template/validate/system_invalid_input.yaml

@@ -0,0 +1,24 @@
+tosca_definitions_version: tosca_simple_yaml_1_0
+
+imports:
+  - queuingsubsystem_invalid_input.yaml
+
+topology_template:
+  description: Test template with invalid input.
+
+  inputs:
+    mq_server_ip:
+      type: string
+      default: 127.0.0.1
+      description: IP address of the message queuing server to receive messages from.
+    mq_server_port:
+      type: integer
+      default: 8080
+      description: Port to be used for receiving messages.
+
+  node_templates:
+    mq:
+      type: example.QueuingSubsystem
+      properties:
+        server_ip: { get_input: mq_server_ip }
+        server_port: { get_input: mq_server_port }

toscaparser/tests/test_topology_template.py

@@ -12,9 +12,12 @@
 
 import os
 
+from toscaparser.common import exception
+from toscaparser.substitution_mappings import SubstitutionMappings
 from toscaparser.tests.base import TestCase
 from toscaparser.topology_template import TopologyTemplate
 from toscaparser.tosca_template import ToscaTemplate
+from toscaparser.utils.gettextutils import _
 import toscaparser.utils.yamlparser
 
 YAML_LOADER = toscaparser.utils.yamlparser.load_yaml
@@ -52,6 +55,18 @@ class TopologyTemplateTest(TestCase):
         custom_defs.update(self._get_custom_def('capability_types'))
         return custom_defs
 
+    def _get_custom_types(self):
+        custom_types = {}
+        def_file = os.path.join(
+            os.path.dirname(os.path.abspath(__file__)),
+            "data/topology_template/definitions.yaml")
+        custom_type = YAML_LOADER(def_file)
+        node_types = custom_type['node_types']
+        for name in node_types:
+            defintion = node_types[name]
+            custom_types[name] = defintion
+        return custom_types
+
     def test_description(self):
         expected_desc = 'Template of a database including its hosting stack.'
         self.assertEqual(expected_desc, self.topo.description)
@@ -162,3 +177,88 @@ class TopologyTemplateTest(TestCase):
         self.assertEqual(
             len(system_tosca_template.
                 nested_tosca_templates_with_topology), 4)
+        self.assertTrue(system_tosca_template.has_nested_templates())
+
+    def test_invalid_keyname(self):
+        tpl_snippet = '''
+        substitution_mappings:
+          node_type: example.DatabaseSubsystem
+          capabilities:
+            database_endpoint: [ db_app, database_endpoint ]
+          requirements:
+            receiver1: [ tran_app, receiver1 ]
+          invalid_key: 123
+        '''
+        sub_mappings = (toscaparser.utils.yamlparser.
+                        simple_parse(tpl_snippet))['substitution_mappings']
+        expected_message = _(
+            'SubstitutionMappings contains unknown field '
+            '"invalid_key". Refer to the definition '
+            'to verify valid values.')
+        err = self.assertRaises(
+            exception.UnknownFieldError,
+            lambda: SubstitutionMappings(sub_mappings, None, None,
+                                         None, None, None))
+        self.assertEqual(expected_message, err.__str__())
+
+    def test_missing_required_keyname(self):
+        tpl_snippet = '''
+        substitution_mappings:
+          capabilities:
+            database_endpoint: [ db_app, database_endpoint ]
+          requirements:
+            receiver1: [ tran_app, receiver1 ]
+        '''
+        sub_mappings = (toscaparser.utils.yamlparser.
+                        simple_parse(tpl_snippet))['substitution_mappings']
+        expected_message = _('SubstitutionMappings used in topology_template '
+                             'is missing required field "node_type".')
+        err = self.assertRaises(
+            exception.MissingRequiredFieldError,
+            lambda: SubstitutionMappings(sub_mappings, None, None,
+                                         None, None, None))
+        self.assertEqual(expected_message, err.__str__())
+
+    def test_invalid_nodetype(self):
+        tpl_snippet = '''
+        substitution_mappings:
+          node_type: example.DatabaseSubsystem1
+          capabilities:
+            database_endpoint: [ db_app, database_endpoint ]
+          requirements:
+            receiver1: [ tran_app, receiver1 ]
+        '''
+        sub_mappings = (toscaparser.utils.yamlparser.
+                        simple_parse(tpl_snippet))['substitution_mappings']
+        custom_defs = self._get_custom_types()
+        expected_message = _('Node type "example.DatabaseSubsystem1" '
+                             'is not a valid type.')
+        err = self.assertRaises(
+            exception.InvalidNodeTypeError,
+            lambda: SubstitutionMappings(sub_mappings, None, None,
+                                         None, None, custom_defs))
+        self.assertEqual(expected_message, err.__str__())
+
+    def test_system_with_input_validation(self):
+        tpl_path0 = os.path.join(
+            os.path.dirname(os.path.abspath(__file__)),
+            "data/topology_template/validate/system_invalid_input.yaml")
+        tpl_path1 = os.path.join(
+            os.path.dirname(os.path.abspath(__file__)),
+            "data/topology_template/validate/"
+            "queuingsubsystem_invalid_input.yaml")
+        errormsg = _('SubstitutionMappings with node_type '
+                     'example.QueuingSubsystem is missing '
+                     'required input definition of input "server_port".')
+
+        # It's invalid in nested template.
+        self.assertRaises(exception.ValidationError,
+                          lambda: ToscaTemplate(tpl_path0))
+        exception.ExceptionCollector.assertExceptionMessage(
+            exception.MissingRequiredInputError, errormsg)
+
+        # Subtemplate deploy standaolone is also invalid.
+        self.assertRaises(exception.ValidationError,
+                          lambda: ToscaTemplate(tpl_path1))
+        exception.ExceptionCollector.assertExceptionMessage(
+            exception.MissingRequiredInputError, errormsg)

toscaparser/tosca_template.py

@@ -14,6 +14,7 @@
 import logging
 import os
 
+from copy import deepcopy
 from toscaparser.common.exception import ExceptionCollector
 from toscaparser.common.exception import InvalidTemplateVersion
 from toscaparser.common.exception import MissingRequiredFieldError
@@ -224,12 +225,14 @@ class ToscaTemplate(object):
         for fname, tosca_tpl in self.nested_tosca_tpls_with_topology.items():
             for nodetemplate in self.nodetemplates:
                 if self._is_sub_mapped_node(nodetemplate, tosca_tpl):
+                    parsed_params = self._get_params_for_nested_template(
+                        nodetemplate)
                     topology_tpl = tosca_tpl.get(TOPOLOGY_TEMPLATE)
                     topology_with_sub_mapping = TopologyTemplate(
                         topology_tpl,
                         self._get_all_custom_defs(),
                         self.relationship_types,
-                        self.parsed_params,
+                        parsed_params,
                         nodetemplate)
                     if topology_with_sub_mapping.substitution_mappings:
                         # Record nested topo templates in top level template
@@ -311,13 +314,23 @@ class ToscaTemplate(object):
         else:
             return False
 
+    def _get_params_for_nested_template(self, nodetemplate):
+        """Return total params for nested_template."""
+        parsed_params = deepcopy(self.parsed_params) \
+            if self.parsed_params else {}
+        if nodetemplate:
+            for pname in nodetemplate.get_properties():
+                parsed_params.update({pname:
+                                      nodetemplate.get_property_value(pname)})
+        return parsed_params
+
     def get_sub_mapping_node_type(self, tosca_tpl):
         """Return substitution mappings node type."""
         if tosca_tpl:
             return TopologyTemplate.get_sub_mapping_node_type(
                 tosca_tpl.get(TOPOLOGY_TEMPLATE))
 
-    def has_substitution_mappings(self):
+    def _has_substitution_mappings(self):
         """Return True if the template has valid substitution mappings."""
         return self.topology_template is not None and \
             self.topology_template.substitution_mappings is not None

toscaparser/utils/gettextutils.py

@@ -19,4 +19,5 @@ _t = gettext.translation('tosca-parser', localedir=_localedir,
 
 
 def _(msg):
+    # type: (object) -> object
     return _t.gettext(msg)