Implement Block Storage Support - Part 1

This patch adds support to attach volumes to VDU.
This patch does not deal with boot from volume case.

Change-Id: I4bee80610eaa4e2ec8a1ccb2a5d36fb723a09806
Implements: persistent-block-storage

doc/source/devref/block_storage_usage_guide.rst

@@ -0,0 +1,132 @@
+..
+  Licensed under the Apache License, Version 2.0 (the "License"); you may
+  not use this file except in compliance with the License. You may obtain
+  a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+  License for the specific language governing permissions and limitations
+  under the License.
+
+.. _ref-vnfd:
+
+=========================================
+Orchestrating VNFs with attached Volumes
+=========================================
+
+To support persistent volumes to VNF, TOSCA NFV profile supports new type
+of nodes. Tacker has now feature of parsing of those new nodes and creation
+of cinder volumes  which are attached to the VDUs.
+
+
+Prerequisites
+~~~~~~~~~~~~~
+To have persistent volume support to VDUs, we must enable cinder service in
+addition to the other services that needed by Tacker.
+
+VNFD Changes
+~~~~~~~~~~~~
+
+There are two steps to have volume attached to VDU:
+
+* Create volume
+* Attach Volume to VDU
+
+Create Volume
+~~~~~~~~~~~~~
+
+To add volume, we need to add the below node to the VNFD:
+
+::
+
+  VB1:
+    type: tosca.nodes.BlockStorage.Tacker
+    properties:
+      size: 1 GB
+
+Attach volume to VDU
+~~~~~~~~~~~~~~~~~~~~
+Next attach the created volume to VDU as below:
+
+::
+
+  CB1:
+    type: tosca.nodes.BlockStorageAttachment
+    properties:
+      location: /dev/vdb
+      requirements:
+        - virtualBinding:
+            node: VDU1
+        - virtualAttachment:
+            node: VB1
+
+With these additions, the new VNFD looks like below:
+
+::
+  tosca_definitions_version: tosca_simple_profile_for_nfv_1_0_0
+  description: Demo example
+
+  metadata:
+    template_name: sample-tosca-vnfd
+
+  topology_template:
+    node_templates:
+      VDU1:
+        type: tosca.nodes.nfv.VDU.Tacker
+        capabilities:
+
+          nfv_compute:
+            properties:
+              num_cpus: 1
+              mem_size: 512 MB
+              disk_size: 1 GB
+
+        properties:
+          image: cirros-0.3.5-x86_64-disk
+          availability_zone: nova
+          mgmt_driver: noop
+
+          config: |
+            param0: key1
+            param1: key2
+
+      CP1:
+        type: tosca.nodes.nfv.CP.Tacker
+
+        properties:
+          management: true
+          order: 0
+          anti_spoofing_protection: false
+        requirements:
+          - virtualLink:
+              node: VL1
+          - virtualBinding:
+              node: VDU1
+
+      VB1:
+        type: tosca.nodes.BlockStorage.Tacker
+
+        properties:
+          size: 1 GB
+
+      CB1:
+        type: tosca.nodes.BlockStorageAttachment
+
+        properties:
+          location: /dev/vdb
+
+          requirements:
+            - virtualBinding:
+                node: VDU1
+            - virtualAttachment:
+                node: VB1
+
+      VL1:
+        type: tosca.nodes.nfv.VL
+
+        properties:
+          network_name: net_mgmt
+          vendor: Tacker

releasenotes/notes/add-block-storage-support-to-tacker-5c349ba112a3c087.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    Add support to attach cinder volumes to the VNF. We can create cinder volumes
+    through new syntax in TOSCA VNFD template and we can attach it to VNFs.
+other:
+  - |
+    Only attaching volumes to the VNFs is supported. Boot from volume is not supported

samples/tosca-templates/vnfd/tosca-vnfd-block-attach.yaml

@@ -0,0 +1,57 @@
+tosca_definitions_version: tosca_simple_profile_for_nfv_1_0_0
+
+description: Demo example
+
+metadata:
+  template_name: sample-tosca-vnfd
+
+topology_template:
+  node_templates:
+    VDU1:
+      type: tosca.nodes.nfv.VDU.Tacker
+      capabilities:
+        nfv_compute:
+          properties:
+            num_cpus: 1
+            mem_size: 512 MB
+            disk_size: 1 GB
+      properties:
+        image: cirros-0.3.5-x86_64-disk
+        availability_zone: nova
+        mgmt_driver: noop
+        config: |
+          param0: key1
+          param1: key2
+
+    CP1:
+      type: tosca.nodes.nfv.CP.Tacker
+      properties:
+        management: true
+        order: 0
+        anti_spoofing_protection: false
+      requirements:
+        - virtualLink:
+            node: VL1
+        - virtualBinding:
+            node: VDU1
+
+    VB1:
+      type: tosca.nodes.BlockStorage.Tacker
+      properties:
+        size: 1 GB
+        image: cirros-0.3.5-x86_64-disk
+
+    CB1:
+          type: tosca.nodes.BlockStorageAttachment
+          properties:
+            location: /dev/vdb
+          requirements:
+            - virtualBinding:
+                node: VDU1
+            - virtualAttachment:
+                node: VB1
+    VL1:
+      type: tosca.nodes.nfv.VL
+      properties:
+        network_name: net_mgmt
+        vendor: Tacker

tacker/tests/etc/samples/sample-tosca-vnfd-block-storage.yaml

@@ -0,0 +1,59 @@
+tosca_definitions_version: tosca_simple_profile_for_nfv_1_0_0
+
+description: Demo example
+
+metadata:
+  template_name: sample-tosca-vnfd
+
+topology_template:
+  node_templates:
+    VDU1:
+      type: tosca.nodes.nfv.VDU.Tacker
+      capabilities:
+        nfv_compute:
+          properties:
+            num_cpus: 1
+            mem_size: 512 MB
+            disk_size: 1 GB
+      properties:
+        name: test-vdu-block-storage
+        image: cirros-0.3.5-x86_64-disk
+        availability_zone: nova
+        mgmt_driver: noop
+        config: |
+          param0: key1
+          param1: key2
+
+    CP1:
+      type: tosca.nodes.nfv.CP.Tacker
+      properties:
+        name: test-cp
+        management: true
+        order: 0
+        anti_spoofing_protection: false
+      requirements:
+        - virtualLink:
+            node: VL1
+        - virtualBinding:
+            node: VDU1
+
+    VB1:
+      type: tosca.nodes.BlockStorage.Tacker
+      properties:
+        size: 1 GB
+        image: cirros-0.3.5-x86_64-disk
+
+    CB1:
+          type: tosca.nodes.BlockStorageAttachment
+          properties:
+            location: /dev/vdb
+          requirements:
+            - virtualBinding:
+                node: VDU1
+            - virtualAttachment:
+                node: VB1
+    VL1:
+      type: tosca.nodes.nfv.VL
+      properties:
+        network_name: net_mgmt
+        vendor: Tacker

tacker/tests/functional/vnfm/test_tosca_vnf_block_storage.py

@@ -0,0 +1,134 @@
+# Copyright 2016 Brocade Communications System, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import yaml
+
+from oslo_config import cfg
+
+from tacker.plugins.common import constants as evt_constants
+from tacker.tests import constants
+from tacker.tests.functional import base
+from tacker.tests.utils import read_file
+
+
+CONF = cfg.CONF
+VNF_CIRROS_CREATE_TIMEOUT = 120
+
+
+class VnfBlockStorageTestToscaCreate(base.BaseTackerTest):
+    def _test_create_vnf(self, vnfd_file, vnf_name,
+                         template_source="onboarded"):
+        data = dict()
+        values_str = read_file(vnfd_file)
+        data['tosca'] = values_str
+        toscal = data['tosca']
+        tosca_arg = {'vnfd': {'name': vnf_name,
+                              'attributes': {'vnfd': toscal}}}
+
+        if template_source == "onboarded":
+            # Create vnfd with tosca template
+            vnfd_instance = self.client.create_vnfd(body=tosca_arg)
+            self.assertIsNotNone(vnfd_instance)
+
+            # Create vnf with vnfd_id
+            vnfd_id = vnfd_instance['vnfd']['id']
+            vnf_arg = {'vnf': {'vnfd_id': vnfd_id, 'name': vnf_name}}
+            vnf_instance = self.client.create_vnf(body=vnf_arg)
+            self.validate_vnf_instance(vnfd_instance, vnf_instance)
+
+        if template_source == 'inline':
+            # create vnf directly from template
+            template = yaml.safe_load(values_str)
+            vnf_arg = {'vnf': {'vnfd_template': template, 'name': vnf_name}}
+            vnf_instance = self.client.create_vnf(body=vnf_arg)
+            vnfd_id = vnf_instance['vnf']['vnfd_id']
+
+        vnf_id = vnf_instance['vnf']['id']
+        self.wait_until_vnf_active(
+            vnf_id,
+            constants.VNF_CIRROS_CREATE_TIMEOUT,
+            constants.ACTIVE_SLEEP_TIME)
+        vnf_show_out = self.client.show_vnf(vnf_id)['vnf']
+        self.assertIsNotNone(vnf_show_out['mgmt_url'])
+
+        input_dict = yaml.safe_load(values_str)
+        prop_dict = input_dict['topology_template']['node_templates'][
+            'CP1']['properties']
+
+        # Verify if ip_address is static, it is same as in show_vnf
+        if prop_dict.get('ip_address'):
+            mgmt_url_input = prop_dict.get('ip_address')
+            mgmt_info = yaml.safe_load(
+                vnf_show_out['mgmt_url'])
+            self.assertEqual(mgmt_url_input, mgmt_info['VDU1'])
+
+        # Verify anti spoofing settings
+        stack_id = vnf_show_out['instance_id']
+        template_dict = input_dict['topology_template']['node_templates']
+        for field in template_dict.keys():
+            prop_dict = template_dict[field]['properties']
+            if prop_dict.get('anti_spoofing_protection'):
+                self.verify_antispoofing_in_stack(stack_id=stack_id,
+                                                  resource_name=field)
+
+        self.verify_vnf_crud_events(
+            vnf_id, evt_constants.RES_EVT_CREATE,
+            evt_constants.PENDING_CREATE, cnt=2)
+        self.verify_vnf_crud_events(
+            vnf_id, evt_constants.RES_EVT_CREATE, evt_constants.ACTIVE)
+        return vnfd_id, vnf_id
+
+    def _test_delete_vnf(self, vnf_id):
+        # Delete vnf_instance with vnf_id
+        try:
+            self.client.delete_vnf(vnf_id)
+        except Exception:
+            assert False, "vnf Delete failed"
+
+        self.wait_until_vnf_delete(vnf_id,
+                                   constants.VNF_CIRROS_DELETE_TIMEOUT)
+        self.verify_vnf_crud_events(vnf_id, evt_constants.RES_EVT_DELETE,
+                                    evt_constants.PENDING_DELETE, cnt=2)
+
+    def _test_cleanup_vnfd(self, vnfd_id, vnf_id):
+        # Delete vnfd_instance
+        self.addCleanup(self.client.delete_vnfd, vnfd_id)
+        self.addCleanup(self.wait_until_vnf_delete, vnf_id,
+            constants.VNF_CIRROS_DELETE_TIMEOUT)
+
+    def _test_create_delete_vnf_tosca(self, vnfd_file, vnf_name,
+            template_source):
+        vnfd_id, vnf_id = self._test_create_vnf(vnfd_file, vnf_name,
+                                                template_source)
+        servers = self.novaclient().servers.list()
+        vdus = []
+        for server in servers:
+            vdus.append(server.name)
+        self.assertIn('test-vdu-block-storage', vdus)
+
+        for server in servers:
+            if server.name == 'test-vdu-block-storage':
+                server_id = server.id
+                server_volumes = self.novaclient().volumes\
+                    .get_server_volumes(server_id)
+                self.assertTrue(len(server_volumes) > 0)
+        self._test_delete_vnf(vnf_id)
+        if template_source == "onboarded":
+            self._test_cleanup_vnfd(vnfd_id, vnf_id)
+
+    def test_create_delete_vnf_tosca_from_vnfd(self):
+        self._test_create_delete_vnf_tosca(
+            'sample-tosca-vnfd-block-storage.yaml',
+            'test_tosca_vnf_with_cirros',
+            'onboarded')

tacker/tests/unit/vnfm/infra_drivers/openstack/data/tosca_block_storage.yaml

@@ -0,0 +1,57 @@
+tosca_definitions_version: tosca_simple_profile_for_nfv_1_0_0
+
+description: Demo example
+
+metadata:
+  template_name: sample-tosca-vnfd
+
+topology_template:
+  node_templates:
+    VDU1:
+      type: tosca.nodes.nfv.VDU.Tacker
+      capabilities:
+        nfv_compute:
+          properties:
+            num_cpus: 1
+            mem_size: 512 MB
+            disk_size: 1 GB
+      properties:
+        image: cirros-0.3.5-x86_64-disk
+        availability_zone: nova
+        mgmt_driver: noop
+        config: |
+          param0: key1
+          param1: key2
+
+    CP1:
+      type: tosca.nodes.nfv.CP.Tacker
+      properties:
+        management: true
+        order: 0
+        anti_spoofing_protection: false
+      requirements:
+        - virtualLink:
+            node: VL1
+        - virtualBinding:
+            node: VDU1
+
+    VB1:
+      type: tosca.nodes.BlockStorage.Tacker
+      properties:
+        size: 1 GB
+        image: cirros-0.3.5-x86_64-disk
+
+    CB1:
+          type: tosca.nodes.BlockStorageAttachment
+          properties:
+            location: /dev/vdb
+          requirements:
+            - virtualBinding:
+                node: VDU1
+            - virtualAttachment:
+                node: VB1
+    VL1:
+      type: tosca.nodes.nfv.VL
+      properties:
+        network_name: net_mgmt
+        vendor: Tacker

tacker/tests/unit/vnfm/tosca/test_utils.py

@@ -25,7 +25,7 @@ from translator.hot import tosca_translator
 def _get_template(name):
     filename = os.path.join(
         os.path.dirname(os.path.abspath(__file__)),
-        "infra_drivers/openstack/data/", name)
+        "../infra_drivers/openstack/data/", name)
     f = codecs.open(filename, encoding='utf-8', errors='strict')
     return f.read()
 
@@ -110,7 +110,7 @@ class TestToscaUtils(testtools.TestCase):
         expected_heat_tpl = _get_template('hot_tosca_openwrt.yaml')
         mgmt_ports = toscautils.get_mgmt_ports(self.tosca)
         heat_tpl = toscautils.post_process_heat_template(
-            heat_template_yaml, mgmt_ports, {}, {})
+            heat_template_yaml, mgmt_ports, {}, {}, {})
 
         heatdict = yaml.safe_load(heat_tpl)
         expecteddict = yaml.safe_load(expected_heat_tpl)
@@ -246,3 +246,23 @@ class TestToscaUtils(testtools.TestCase):
         toscautils.updateimports(template)
         toscautils.check_for_substitution_mappings(template, param)
         self.assertNotIn('substitution_mappings', param)
+
+    def test_get_block_storage_details(self):
+        tosca_vol = _get_template('tosca_block_storage.yaml')
+        vnfd_dict = yaml.safe_load(tosca_vol)
+        expected_dict = {
+            'volumes': {
+                'VB1': {
+                    'image': 'cirros-0.3.5-x86_64-disk',
+                    'size': '1'
+                }
+            },
+            'volume_attachments': {
+                'CB1': {
+                    'instance_uuid': {'get_resource': 'VDU1'},
+                    'mountpoint': '/dev/vdb',
+                    'volume_id': {'get_resource': 'VB1'}}
+            }
+        }
+        volume_details = toscautils.get_block_storage_details(vnfd_dict)
+        self.assertEqual(expected_dict, volume_details)

tacker/tosca/lib/tacker_nfv_defs.yaml

@@ -268,3 +268,22 @@ node_types:
       - host:
          node: tosca.nodes.nfv.VDU.Tacker
          relationship: tosca.relationships.HostedOn
+
+  tosca.nodes.BlockStorage.Tacker:
+    derived_from: tosca.nodes.BlockStorage
+    properties:
+      image:
+        type: string
+        required: false
+
+  tosca.nodes.BlockStorageAttachment:
+    derived_from: tosca.nodes.Root
+    properties:
+      location:
+        type: string
+        required: true
+    requirements:
+      - virtualBinding:
+          node: tosca.nodes.nfv.VDU.Tacker
+      - virtualAttachment:
+          node: tosca.nodes.BlockStorage.Tacker

tacker/tosca/utils.py

@@ -34,6 +34,8 @@ SCALING = 'tosca.policies.Scaling'
 PLACEMENT = 'tosca.policies.tacker.Placement'
 TACKERCP = 'tosca.nodes.nfv.CP.Tacker'
 TACKERVDU = 'tosca.nodes.nfv.VDU.Tacker'
+BLOCKSTORAGE = 'tosca.nodes.BlockStorage.Tacker'
+BLOCKSTORAGE_ATTACHMENT = 'tosca.nodes.BlockStorageAttachment'
 TOSCA_BINDS_TO = 'tosca.relationships.network.BindsTo'
 VDU = 'tosca.nodes.nfv.VDU'
 IMAGE = 'tosca.artifacts.Deployment.Image.VM'
@@ -219,6 +221,62 @@ def _process_alarm_actions(vnf, policy):
     return alarm_actions
 
 
+def get_volumes(template):
+    volume_dict = dict()
+    node_tpl = template['topology_template']['node_templates']
+    for node_name in list(node_tpl.keys()):
+        node_value = node_tpl[node_name]
+        if node_value['type'] != BLOCKSTORAGE:
+            continue
+        volume_dict[node_name] = dict()
+        block_properties = node_value.get('properties', {})
+        for prop_name, prop_value in block_properties.items():
+            if prop_name == 'size':
+                prop_value = \
+                    re.compile('(\d+)\s*(\w+)').match(prop_value).groups()[0]
+            volume_dict[node_name][prop_name] = prop_value
+        del node_tpl[node_name]
+    return volume_dict
+
+
+@log.log
+def get_vol_attachments(template):
+    vol_attach_dict = dict()
+    node_tpl = template['topology_template']['node_templates']
+    valid_properties = {
+        'location': 'mountpoint'
+    }
+    for node_name in list(node_tpl.keys()):
+        node_value = node_tpl[node_name]
+        if node_value['type'] != BLOCKSTORAGE_ATTACHMENT:
+            continue
+        vol_attach_dict[node_name] = dict()
+        vol_attach_properties = node_value.get('properties', {})
+        # parse properties
+        for prop_name, prop_value in vol_attach_properties.items():
+            if prop_name in valid_properties:
+                vol_attach_dict[node_name][valid_properties[prop_name]] = \
+                    prop_value
+        # parse requirements to get mapping of cinder volume <-> Nova instance
+        for req in node_value.get('requirements', {}):
+            if 'virtualBinding' in req:
+                vol_attach_dict[node_name]['instance_uuid'] = \
+                    {'get_resource': req['virtualBinding']['node']}
+            elif 'virtualAttachment' in req:
+                vol_attach_dict[node_name]['volume_id'] = \
+                    {'get_resource': req['virtualAttachment']['node']}
+        del node_tpl[node_name]
+    return vol_attach_dict
+
+
+@log.log
+def get_block_storage_details(template):
+    block_storage_details = dict()
+    block_storage_details['volumes'] = get_volumes(template)
+    block_storage_details['volume_attachments'] = get_vol_attachments(template)
+    return block_storage_details
+
+
 @log.log
 def get_mgmt_ports(tosca):
     mgmt_ports = {}
@@ -308,7 +366,7 @@ def represent_odict(dump, tag, mapping, flow_style=None):
 @log.log
 def post_process_heat_template(heat_tpl, mgmt_ports, metadata,
                                alarm_resources, res_tpl,
-                               unsupported_res_prop=None):
+                               vol_res={}, unsupported_res_prop=None):
     #
     # TODO(bobh) - remove when heat-translator can support literal strings.
     #
@@ -359,6 +417,8 @@ def post_process_heat_template(heat_tpl, mgmt_ports, metadata,
         if 'get_file' in config:
             res["properties"]["config"] = open(config["get_file"]).read()
 
+    if vol_res.get('volumes'):
+        add_volume_resources(heat_dict, vol_res)
     if unsupported_res_prop:
         convert_unsupported_res_prop(heat_dict, unsupported_res_prop)
 
@@ -369,6 +429,28 @@ def post_process_heat_template(heat_tpl, mgmt_ports, metadata,
     return yaml.safe_dump(heat_dict)
 
 
+@log.log
+def add_volume_resources(heat_dict, vol_res):
+    # Add cinder volumes
+    for res_name, cinder_vol in vol_res['volumes'].items():
+        heat_dict['resources'][res_name] = {
+            'type': 'OS::Cinder::Volume',
+            'properties': {}
+        }
+        for prop_name, prop_val in cinder_vol.items():
+            heat_dict['resources'][res_name]['properties'][prop_name] = \
+                prop_val
+    # Add cinder volume attachments
+    for res_name, cinder_vol in vol_res['volume_attachments'].items():
+        heat_dict['resources'][res_name] = {
+            'type': 'OS::Cinder::VolumeAttachment',
+            'properties': {}
+        }
+        for prop_name, prop_val in cinder_vol.items():
+            heat_dict['resources'][res_name]['properties'][prop_name] = \
+                prop_val
+
+
 @log.log
 def post_process_template(template):
     for nt in template.nodetemplates:

tacker/vnfm/infra_drivers/openstack/translate_template.py

@@ -250,6 +250,7 @@ class TOSCAToHOT(object):
                 LOG.debug("Params not Well Formed: %s", str(e))
                 raise vnfm.ParamYAMLNotWellFormed(error_msg_details=str(e))
 
+        block_storage_details = toscautils.get_block_storage_details(vnfd_dict)
         toscautils.updateimports(vnfd_dict)
         if 'substitution_mappings' in str(vnfd_dict):
             toscautils.check_for_substitution_mappings(vnfd_dict,
@@ -308,7 +309,7 @@ class TOSCAToHOT(object):
 
         heat_template_yaml = toscautils.post_process_heat_template(
             heat_template_yaml, mgmt_ports, metadata, alarm_resources,
-            res_tpl, self.unsupported_props)
+            res_tpl, block_storage_details, self.unsupported_props)
 
         self.heat_template_yaml = heat_template_yaml
         self.monitoring_dict = monitoring_dict