Implementation of policies in topology template

Implemented policies in topology template. Corrected groups class and
properties. groups will be added in seperate patch.
As of this patch, corrected key names as per spec.
Added validation of keys for policy and groups.
Added functionality (version, targets, description) to PolicyType class

Added testcases : policies for nodetemplates & policies for groups

Partially Implements: blueprint tosca-policies
Co-Authored-By: madhavi <madhavilatha.t@tcs.com >
Change-Id: I22f8c442274fd0c6d361c4cbd1aeacdda3daaeaa

toscaparser/elements/policytype.py

@@ -10,21 +10,49 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from toscaparser.common.exception import ExceptionCollector
+from toscaparser.common.exception import InvalidTypeError
+from toscaparser.common.exception import UnknownFieldError
 from toscaparser.elements.statefulentitytype import StatefulEntityType
+from toscaparser.utils.validateutils import TOSCAVersionProperty
 
 
 class PolicyType(StatefulEntityType):
+
     '''TOSCA built-in policies type.'''
+    SECTIONS = (DERIVED_FROM, METADATA, PROPERTIES, VERSION, DESCRIPTION, TARGETS) = \
+               ('derived_from', 'metadata', 'properties', 'version',
+                'description', 'targets')
 
     def __init__(self, ptype, custom_def=None):
         super(PolicyType, self).__init__(ptype, self.POLICY_PREFIX,
                                          custom_def)
         self.type = ptype
+        self._validate_keys()
+
+        self.meta_data = None
+        if self.METADATA in self.defs:
+            self.meta_data = self.defs[self.METADATA]
+            self._validate_metadata(self.meta_data)
+
         self.properties = None
         if self.PROPERTIES in self.defs:
             self.properties = self.defs[self.PROPERTIES]
         self.parent_policies = self._get_parent_policies()
 
+        self.policy_version = None
+        if self.VERSION in self.defs:
+            self.policy_version = TOSCAVersionProperty(
+                self.defs[self.VERSION]).get_version()
+
+        self.policy_description = self.defs[self.DESCRIPTION] \
+            if self.DESCRIPTION in self.defs else None
+
+        self.targets_list = None
+        if self.TARGETS in self.defs:
+            self.targets_list = self.defs[self.TARGETS]
+            self._validate_targets(self.targets_list, custom_def)
+
     def _get_parent_policies(self):
         policies = {}
         parent_policy = self.parent_type
@@ -43,3 +71,45 @@ class PolicyType(StatefulEntityType):
         '''Return the definition of a policy field by name.'''
         if name in self.defs:
             return self.defs[name]
+
+    @property
+    def targets(self):
+        '''Return targets.'''
+        return self.targets_list
+
+    @property
+    def description(self):
+        return self.policy_description
+
+    @property
+    def version(self):
+        return self.policy_version
+
+    def _validate_keys(self):
+        for key in self.defs.keys():
+            if key not in self.SECTIONS:
+                ExceptionCollector.appendException(
+                    UnknownFieldError(what='Policy "%s"' % self.name,
+                                      field=key))
+
+    def _validate_targets(self, targets_list, custom_def):
+        for nodetype in targets_list:
+            if nodetype not in custom_def:
+                ExceptionCollector.appendException(
+                    InvalidTypeError(what='"%s" defined in targets for '
+                                     'policy "%s"' % (nodetype, self.type)))
+
+    def _validate_metadata(self, meta_data):
+        if not meta_data.get('type') in ['map', 'tosca:map']:
+            ExceptionCollector.appendException(
+                InvalidTypeError(what='"%s" defined in policy for '
+                                 'metadata' % (meta_data.get('type'))))
+
+        for entry_schema, entry_schema_type in meta_data.items():
+            if isinstance(entry_schema_type, dict) and not \
+                entry_schema_type.get('type') == 'string':
+                ExceptionCollector.appendException(
+                    InvalidTypeError(what='"%s" defined in policy for '
+                                     'metadata "%s"'
+                                     % (entry_schema_type.get('type'),
+                                        entry_schema)))

toscaparser/entity_template.py

@@ -17,8 +17,10 @@ from toscaparser.common.exception import UnknownFieldError
 from toscaparser.common.exception import ValidationError
 from toscaparser.elements.interfaces import InterfacesDef
 from toscaparser.elements.nodetype import NodeType
+from toscaparser.elements.policytype import PolicyType
 from toscaparser.elements.relationshiptype import RelationshipType
 from toscaparser.properties import Property
+from toscaparser.utils.gettextutils import _
 
 
 class EntityTemplate(object):
@@ -56,6 +58,15 @@ class EntityTemplate(object):
                 type = self.entity_tpl['type']
             self.type_definition = RelationshipType(type,
                                                     None, custom_def)
+        if entity_name == 'policy_type':
+            type = self.entity_tpl.get('type')
+            if not type:
+                msg = (_('Policy definition of "%(pname)s" must have'
+                       ' a "type" ''attribute.') % dict(pname=name))
+                ExceptionCollector.appendException(
+                    ValidationError(msg))
+
+            self.type_definition = PolicyType(type, custom_def)
         self._properties = None
         self._interfaces = None
         self._requirements = None

toscaparser/groups.py

@@ -11,17 +11,46 @@
 #    under the License.
 
 
-class NodeGroup(object):
+from toscaparser.common.exception import ExceptionCollector
+from toscaparser.common.exception import UnknownFieldError
+from toscaparser.entity_template import EntityTemplate
+from toscaparser.utils import validateutils
+
+SECTIONS = (TYPE, METADATA, DESCRIPTION, PROPERTIES, TARGETS, INTERFACES) = \
+           ('type', 'metadata', 'description',
+            'properties', 'targets', 'interfaces')
+
+
+class Group(EntityTemplate):
 
     def __init__(self, name, group_templates, member_nodes):
+        super(Group, self).__init__(name,
+                                    group_templates,
+                                    'group_type',
+                                    None)
         self.name = name
         self.tpl = group_templates
+        self.meta_data = None
+        if self.METADATA in self.tpl:
+            self.meta_data = self.tpl.get(self.METADATA)
+            validateutils.validate_map(self.meta_data)
         self.members = member_nodes
+        self._validate_keys()
 
     @property
-    def member_names(self):
-        return self.tpl.get('members')
+    def targets(self):
+        return self.tpl.get('targets')
 
     @property
-    def policies(self):
-        return self.tpl.get('policies')
+    def description(self):
+        return self.entity_tpl.get('description')
+
+    def get_members(self):
+        return self.members
+
+    def _validate_keys(self):
+        for key in self.entity_tpl.keys():
+            if key not in SECTIONS:
+                ExceptionCollector.appendException(
+                    UnknownFieldError(what='Groups "%s"' % self.name,
+                                      field=key))

toscaparser/policy.py

@@ -0,0 +1,65 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+import logging
+
+from toscaparser.common.exception import ExceptionCollector
+from toscaparser.common.exception import UnknownFieldError
+from toscaparser.entity_template import EntityTemplate
+from toscaparser.utils import validateutils
+
+SECTIONS = (TYPE, METADATA, DESCRIPTION, PROPERTIES, TARGETS) = \
+           ('type', 'metadata', 'description', 'properties', 'targets')
+
+log = logging.getLogger('tosca')
+
+
+class Policy(EntityTemplate):
+    '''Policies defined in Topology template.'''
+    def __init__(self, name, policy, targets, targets_type, custom_def=None):
+        super(Policy, self).__init__(name,
+                                     policy,
+                                     'policy_type',
+                                     custom_def)
+        self.meta_data = None
+        if self.METADATA in policy:
+            self.meta_data = policy.get(self.METADATA)
+            validateutils.validate_map(self.meta_data)
+        self.targets_list = targets
+        self.targets_type = targets_type
+        self._validate_keys()
+
+    @property
+    def targets(self):
+        return self.entity_tpl.get('targets')
+
+    @property
+    def description(self):
+        return self.entity_tpl.get('description')
+
+    @property
+    def metadata(self):
+        return self.entity_tpl.get('metadata')
+
+    def get_targets_type(self):
+        return self.targets_type
+
+    def get_targets_list(self):
+        return self.targets_list
+
+    def _validate_keys(self):
+        for key in self.entity_tpl.keys():
+            if key not in SECTIONS:
+                ExceptionCollector.appendException(
+                    UnknownFieldError(what='Policy "%s"' % self.name,
+                                      field=key))

toscaparser/tests/data/policies/custom_definitions.yaml

@@ -0,0 +1,10 @@
+tosca_definitions_version: tosca_simple_yaml_1_0
+
+policy_types:
+  mycompany.mytypes.myScalingPolicy:
+    derived_from: tosca.policies.Scaling
+    metadata:
+     type: map
+     entry_schema:
+       type: string
+

toscaparser/tests/data/policies/tosca_policy_template.yaml

@@ -0,0 +1,64 @@
+tosca_definitions_version: tosca_simple_yaml_1_0
+
+description: >
+  Template for deploying servers based on policies.
+
+imports:
+  - custom_definitions.yaml
+
+topology_template:
+  node_templates:
+    my_server_1:
+      type: tosca.nodes.Compute
+      capabilities:
+        # Host container properties
+        host:
+         properties:
+           num_cpus: 2
+           disk_size: 10 GB
+           mem_size: 512 MB
+        # Guest Operating System properties
+        os:
+          properties:
+            # host Operating System image properties
+            architecture: x86_64
+            type: Linux
+            distribution: RHEL
+            version: 6.5
+
+    my_server_2:
+      type: tosca.nodes.Compute
+      capabilities:
+        host:
+          properties:
+            disk_size: 10 GB
+            num_cpus: 2
+            mem_size: 4096 MB
+        os:
+          properties:
+            architecture: x86_64
+            type: Linux
+            distribution: Ubuntu
+            version: 14.04
+
+  groups:
+    webserver_group:
+      targets: [ my_server_1, my_server_2 ]
+      type: tosca.groups.Root
+      metadata: { user1: 1008, user2: 1002 }
+
+
+  policies:
+    - my_compute_placement_policy:
+        type: tosca.policies.Placement
+        description: Apply placement policy to servers
+        metadata: { user1: 1001, user2: 1002 }
+        targets: [ my_server_1, my_server_2 ]
+    - my_groups_placement:
+        type: mycompany.mytypes.myScalingPolicy
+        targets: [ webserver_group ]
+        description: my company scaling policy
+        metadata:
+          user1: 1001
+          user2: 1003
+

toscaparser/tests/data/topology_template/definitions.yaml

@@ -51,4 +51,3 @@ capability_types:
      properties:
        server_ip:
          type: string
-

toscaparser/tests/data/topology_template/subsystem.yaml

@@ -79,7 +79,6 @@ topology_template:
 
   groups:
     webserver_group:
-      members: [ websrv, server ]
-      policies:
-        - policy_name: none
+      targets: [ websrv, server ]
+      type: tosca.groups.Root
 

toscaparser/tests/test_topology_template.py

@@ -145,7 +145,7 @@ class TopologyTemplateTest(TestCase):
     def test_groups(self):
         group = self.topo.groups[0]
         self.assertEqual('webserver_group', group.name)
-        self.assertEqual(['websrv', 'server'], group.member_names)
+        self.assertEqual(['websrv', 'server'], group.targets)
         for node in group.members:
             if node.name == 'server':
                 '''Test property value'''

toscaparser/tests/test_toscatpl.py

@@ -616,3 +616,44 @@ class ToscaTemplateTest(TestCase):
                      'cannot be used in a pre-parsed input template.'))
         exception.ExceptionCollector.assertExceptionMessage(ImportError,
                                                             err_msg)
+
+    def test_policies_for_node_templates(self):
+        tosca_tpl = os.path.join(
+            os.path.dirname(os.path.abspath(__file__)),
+            "data/policies/tosca_policy_template.yaml")
+        tosca = ToscaTemplate(tosca_tpl)
+
+        for policy in tosca.topology_template.policies:
+            if policy.name == 'my_compute_placement_policy':
+                self.assertEqual('tosca.policies.Placement', policy.type)
+                self.assertEqual(['my_server_1', 'my_server_2'],
+                                 policy.targets)
+                self.assertEqual('node_templates', policy.get_targets_type())
+                for node in policy.targets_list:
+                    if node.name == 'my_server_1':
+                        '''Test property value'''
+                        props = node.get_properties()
+                        if props and 'mem_size' in props.keys():
+                            self.assertEqual(props['mem_size'].value,
+                                             '4096 MB')
+
+    def test_policies_for_groups(self):
+        tosca_tpl = os.path.join(
+            os.path.dirname(os.path.abspath(__file__)),
+            "data/policies/tosca_policy_template.yaml")
+        tosca = ToscaTemplate(tosca_tpl)
+
+        for policy in tosca.topology_template.policies:
+            if policy.name == 'my_groups_placement':
+                self.assertEqual('mycompany.mytypes.myScalingPolicy',
+                                 policy.type)
+                self.assertEqual(['webserver_group'], policy.targets)
+                self.assertEqual('groups', policy.get_targets_type())
+                group = policy.get_targets_list()[0]
+                for node in group.get_members():
+                    if node.name == 'my_server_2':
+                        '''Test property value'''
+                        props = node.get_properties()
+                        if props and 'mem_size' in props.keys():
+                            self.assertEqual(props['mem_size'].value,
+                                             '4096 MB')

toscaparser/tests/test_toscatplvalidation.py

@@ -18,6 +18,7 @@ from toscaparser.imports import ImportsLoader
 from toscaparser.nodetemplate import NodeTemplate
 from toscaparser.parameters import Input
 from toscaparser.parameters import Output
+from toscaparser.policy import Policy
 from toscaparser.relationship_template import RelationshipTemplate
 from toscaparser.tests.base import TestCase
 from toscaparser.tosca_template import ToscaTemplate
@@ -1119,3 +1120,53 @@ custom_types/wordpress.yaml
             metadata: none
         '''
         self._single_node_template_content_test(tpl_snippet_metadata_inline)
+
+    def test_policy_valid_keynames(self):
+        tpl_snippet = '''
+        policies:
+          - servers_placement:
+              type: tosca.policies.Placement
+              description: Apply placement policy to servers
+              metadata: { user1: 1001, user2: 1002 }
+              targets: [ serv1, serv2 ]
+        '''
+        policies = (toscaparser.utils.yamlparser.
+                    simple_parse(tpl_snippet))['policies'][0]
+        name = list(policies.keys())[0]
+        Policy(name, policies[name], None, None)
+
+    def test_policy_invalid_keyname(self):
+        tpl_snippet = '''
+        policies:
+          - servers_placement:
+              type: tosca.policies.Placement
+              testkey: testvalue
+        '''
+        policies = (toscaparser.utils.yamlparser.
+                    simple_parse(tpl_snippet))['policies'][0]
+        name = list(policies.keys())[0]
+
+        expectedmessage = _('Policy "servers_placement" contains '
+                            'unknown field "testkey". Refer to the '
+                            'definition to verify valid values.')
+        err = self.assertRaises(
+            exception.UnknownFieldError,
+            lambda: Policy(name, policies[name], None, None))
+        self.assertEqual(expectedmessage, err.__str__())
+
+    def test_policy_missing_required_keyname(self):
+        tpl_snippet = '''
+        policies:
+          - servers_placement:
+              description: test description
+        '''
+        policies = (toscaparser.utils.yamlparser.
+                    simple_parse(tpl_snippet))['policies'][0]
+        name = list(policies.keys())[0]
+
+        expectedmessage = _('Template "servers_placement" is missing '
+                            'required field "type".')
+        err = self.assertRaises(
+            exception.MissingRequiredFieldError,
+            lambda: Policy(name, policies[name], None, None))
+        self.assertEqual(expectedmessage, err.__str__())

toscaparser/topology_template.py

@@ -15,10 +15,11 @@ import logging
 
 from toscaparser.common import exception
 from toscaparser import functions
-from toscaparser.groups import NodeGroup
+from toscaparser.groups import Group
 from toscaparser.nodetemplate import NodeTemplate
 from toscaparser.parameters import Input
 from toscaparser.parameters import Output
+from toscaparser.policy import Policy
 from toscaparser.relationship_template import RelationshipTemplate
 from toscaparser.tpl_relationship_graph import ToscaGraph
 
@@ -26,10 +27,10 @@ from toscaparser.tpl_relationship_graph import ToscaGraph
 # Topology template key names
 SECTIONS = (DESCRIPTION, INPUTS, NODE_TEMPLATES,
             RELATIONSHIP_TEMPLATES, OUTPUTS, GROUPS,
-            SUBSTITUION_MAPPINGS) = \
+            SUBSTITUION_MAPPINGS, POLICIES) = \
            ('description', 'inputs', 'node_templates',
             'relationship_templates', 'outputs', 'groups',
-            'substitution_mappings')
+            'substitution_mappings', 'policies')
 
 log = logging.getLogger("tosca.model")
 
@@ -53,6 +54,7 @@ class TopologyTemplate(object):
             if hasattr(self, 'nodetemplates'):
                 self.graph = ToscaGraph(self.nodetemplates)
             self.groups = self._groups()
+            self.policies = self._policies()
             self._process_intrinsic_functions()
 
     def _inputs(self):
@@ -99,19 +101,37 @@ class TopologyTemplate(object):
     def _substitution_mappings(self):
         pass
 
+    def _policies(self):
+        policies = []
+        for policy in self._tpl_policies():
+            for policy_name, policy_tpl in policy.items():
+                target_list = policy_tpl.get('targets')
+                if target_list and len(target_list) >= 1:
+                    target_objects = []
+                    targets_type = "node_templates"
+                    target_objects = self._get_group_members(target_list)
+                    if not target_objects:
+                        target_objects = self._get_policy_groups(target_list)
+                        targets_type = "groups"
+                    policyObj = Policy(policy_name, policy_tpl,
+                                       target_objects, targets_type,
+                                       self.custom_defs)
+                    policies.append(policyObj)
+        return policies
+
     def _groups(self):
         groups = []
         for group_name, group_tpl in self._tpl_groups().items():
-            member_names = group_tpl.get('members')
+            member_names = group_tpl.get('targets')
             if member_names and len(member_names) > 1:
-                group = NodeGroup(group_name, group_tpl,
-                                  self._get_group_memerbs(member_names))
+                group = Group(group_name, group_tpl,
+                              self._get_group_members(member_names))
                 groups.append(group)
             else:
                 exception.ExceptionCollector.appendException(ValueError)
         return groups
 
-    def _get_group_memerbs(self, member_names):
+    def _get_group_members(self, member_names):
         member_nodes = []
         for member in member_names:
             for node in self.nodetemplates:
@@ -119,6 +139,14 @@ class TopologyTemplate(object):
                     member_nodes.append(node)
         return member_nodes
 
+    def _get_policy_groups(self, member_names):
+        member_groups = []
+        for member in member_names:
+            for group in self.groups:
+                if group.name == member:
+                    member_groups.append(group)
+        return member_groups
+
     # topology template can act like node template
     # it is exposed by substitution_mappings.
     def nodetype(self):
@@ -153,6 +181,9 @@ class TopologyTemplate(object):
     def _tpl_groups(self):
         return self.tpl.get(GROUPS) or {}
 
+    def _tpl_policies(self):
+        return self.tpl.get(POLICIES) or {}
+
     def _validate_field(self):
         for name in self.tpl:
             if name not in SECTIONS:

toscaparser/tosca_template.py

@@ -34,13 +34,14 @@ SECTIONS = (DEFINITION_VERSION, DEFAULT_NAMESPACE, TEMPLATE_NAME,
             TOPOLOGY_TEMPLATE, TEMPLATE_AUTHOR, TEMPLATE_VERSION,
             DESCRIPTION, IMPORTS, DSL_DEFINITIONS, NODE_TYPES,
             RELATIONSHIP_TYPES, RELATIONSHIP_TEMPLATES,
-            CAPABILITY_TYPES, ARTIFACT_TYPES, DATATYPE_DEFINITIONS) = \
+            CAPABILITY_TYPES, ARTIFACT_TYPES, DATATYPE_DEFINITIONS,
+            POLICY_TYPES) = \
            ('tosca_definitions_version', 'tosca_default_namespace',
             'template_name', 'topology_template', 'template_author',
             'template_version', 'description', 'imports', 'dsl_definitions',
             'node_types', 'relationship_types', 'relationship_templates',
-            'capability_types', 'artifact_types', 'datatype_definitions')
-
+            'capability_types', 'artifact_types', 'datatype_definitions',
+            'policy_types')
 # Sections that are specific to individual template definitions
 SPECIAL_SECTIONS = (METADATA) = ('metadata')
 
@@ -144,7 +145,7 @@ class ToscaTemplate(object):
 
     def _get_all_custom_defs(self, imports=None):
         types = [IMPORTS, NODE_TYPES, CAPABILITY_TYPES, RELATIONSHIP_TYPES,
-                 DATATYPE_DEFINITIONS]
+                 DATATYPE_DEFINITIONS, POLICY_TYPES]
         custom_defs_final = {}
         custom_defs = self._get_custom_types(types, imports)
         if custom_defs: