Add share manage/unmanage of Oracle ZFSSA driver

- Manage share: ask Manila to take control of an existing ZFSSA share
not managed by Manila driver. Current share size will be rounded up
to the nearest whole GB value.
- Unmanage share: ask Manila to release control of a ZFSSA share
managed by the driver.

Change-Id: I0d4a46211dfd15cb7a261eddfa8c14be26adcfc3
Implements: blueprint oracle-zfssa-share-manage

doc/source/devref/share_back_ends_feature_support_mapping.rst

@@ -63,7 +63,7 @@ Mapping of share drivers and share features support
 +----------------------------------------+-----------------------+-----------------------+--------------+--------------+------------------------+----------------------------+--------------------------+
 |              Windows SMB               |           L           |           L           |       L      |       L      |            L           |              L             |            \-            |
 +----------------------------------------+-----------------------+-----------------------+--------------+--------------+------------------------+----------------------------+--------------------------+
-|             Oracle ZFSSA               |           K           |          \-           |       M      |       M      |            K           |              K             |            \-            |
+|             Oracle ZFSSA               |           K           |           N           |       M      |       M      |            K           |              K             |            \-            |
 +----------------------------------------+-----------------------+-----------------------+--------------+--------------+------------------------+----------------------------+--------------------------+
 |             CephFS Native              |           M           |          \-           |      M       |      M       |            M           |             \-             |            \-            |
 +----------------------------------------+-----------------------+-----------------------+--------------+--------------+------------------------+----------------------------+--------------------------+

manila/share/drivers/zfssa/zfssarest.py

@@ -20,6 +20,7 @@ from oslo_serialization import jsonutils
 from manila import exception
 from manila.i18n import _
 from manila.i18n import _LE
+from manila.i18n import _LW
 from manila.share.drivers.zfssa import restclient
 
 
@@ -385,3 +386,27 @@ class ZFSSAApi(object):
         arg = {'sharenfs': argval}
         LOG.debug('deny_access: %s', argval)
         self.modify_share(pool, project, share, arg)
+
+    def create_schema(self, schema):
+        """Create a custom ZFSSA schema."""
+        base = '/api/storage/v1/schema'
+        svc = "%(base)s/%(prop)s" % {'base': base, 'prop': schema['property']}
+        ret = self.rclient.get(svc)
+        if ret.status == restclient.Status.OK:
+            LOG.warning(_LW('Property %s already exists.'), schema['property'])
+            return
+        ret = self.rclient.post(base, schema)
+        if ret.status != restclient.Status.CREATED:
+            exception_msg = (_('Error Creating '
+                               'Property: %(property)s '
+                               'Type: %(type)s '
+                               'Description: %(description)s '
+                               'Return code: %(ret.status)d '
+                               'Message: %(ret.data)s.')
+                             % {'property': schema['property'],
+                                'type': schema['type'],
+                                'description': schema['description'],
+                                'ret.status': ret.status,
+                                'ret.data': ret.data})
+            LOG.error(exception_msg)
+            raise exception.ShareBackendException(msg=exception_msg)

manila/share/drivers/zfssa/zfssashare.py

@@ -16,7 +16,7 @@ ZFS Storage Appliance Manila Share Driver
 """
 
 import base64
-
+import math
 from oslo_config import cfg
 from oslo_log import log
 from oslo_utils import units
@@ -57,7 +57,13 @@ ZFSSA_OPTS = [
     cfg.StrOpt('zfssa_nas_vscan', default='false',
                help='Controls whether the share is scanned for viruses.'),
     cfg.StrOpt('zfssa_rest_timeout',
-               help='REST connection timeout (in seconds).')
+               help='REST connection timeout (in seconds).'),
+    cfg.StrOpt('zfssa_manage_policy', default='loose',
+               choices=['loose', 'strict'],
+               help='Driver policy for share manage. A strict policy checks '
+                    'for a schema named manila_managed, and makes sure its '
+                    'value is true. A loose policy does not check for the '
+                    'schema.')
 ]
 
 cfg.CONF.register_opts(ZFSSA_OPTS)
@@ -77,9 +83,10 @@ class ZFSSAShareDriver(driver.ShareDriver):
 
         1.0 - Initial version.
         1.0.1 - Add share shrink/extend feature.
+        1.0.2 - Add share manage/unmanage feature.
     """
 
-    VERSION = '1.0.1'
+    VERSION = '1.0.2'
     PROTOCOL = 'NFS_CIFS'
 
     def __init__(self, *args, **kwargs):
@@ -122,6 +129,7 @@ class ZFSSAShareDriver(driver.ShareDriver):
             'sharesmb': 'off',
             'quota_snap': self.configuration.zfssa_nas_quota_snap,
             'reservation_snap': self.configuration.zfssa_nas_quota_snap,
+            'custom:manila_managed': True,
         }
 
     def do_setup(self, context):
@@ -148,6 +156,13 @@ class ZFSSAShareDriver(driver.ShareDriver):
         self.zfssa.enable_service('nfs')
         self.zfssa.enable_service('smb')
 
+        schema = {
+            'property': 'manila_managed',
+            'description': 'Managed by Manila',
+            'type': 'Boolean',
+        }
+        self.zfssa.create_schema(schema)
+
     def check_for_setup_error(self):
         """Check for properly configured pool, project."""
         lcfg = self.configuration
@@ -270,14 +285,148 @@ class ZFSSAShareDriver(driver.ShareDriver):
                                    snapshot['share_id'],
                                    snapshot['id'])
 
-    def ensure_share(self, context, share, share_server=None):
+    def manage_existing(self, share, driver_options):
+        """Manage an existing ZFSSA share.
+
+        This feature requires an option 'zfssa_name', which specifies the
+        name of the share as appeared in ZFSSA.
+
+        The driver automatically retrieves information from the ZFSSA backend
+        and returns the correct share size and export location.
+        """
+        if 'zfssa_name' not in driver_options:
+            msg = _('Name of the share in ZFSSA share has to be '
+                    'specified in option zfssa_name.')
+            LOG.error(msg)
+            raise exception.ShareBackendException(msg=msg)
+        name = driver_options['zfssa_name']
+        try:
+            details = self._get_share_details(name)
+        except Exception as e:
+            LOG.error(_LE('Cannot manage share %s'), name)
+            raise e
+
+        lcfg = self.configuration
+        input_export_loc = share['export_locations'][0]['path']
+        proto = share['share_proto']
+
+        self._verify_share_to_manage(name, details)
+
+        # Get and verify share size:
+        size_byte = details['quota']
+        size_gb = int(math.ceil(size_byte / float(units.Gi)))
+        if size_byte % units.Gi != 0:
+            # Round up the size:
+            new_size_byte = size_gb * units.Gi
+            free_space = self.zfssa.get_project_stats(lcfg.zfssa_pool,
+                                                      lcfg.zfssa_project)
+
+            diff_space = int(new_size_byte - size_byte)
+
+            if diff_space > free_space:
+                msg = (_('Quota and reservation of share %(name)s need to be '
+                         'rounded up to %(size)d. But there is not enough '
+                         'space in the backend.') % {'name': name,
+                                                     'size': size_gb})
+                LOG.error(msg)
+                raise exception.ManageInvalidShare(reason=msg)
+            size_byte = new_size_byte
+
+        # Get and verify share export location, also update share properties.
+        arg = {
+            'host': lcfg.zfssa_data_ip,
+            'mountpoint': input_export_loc,
+            'name': share['id'],
+        }
+        manage_args = self.default_args.copy()
+        manage_args.update(self.share_args)
+        # The ZFSSA share name has to be updated, as Manila generates a new
+        # share id for each share to be managed.
+        manage_args.update({'name': share['id'],
+                            'quota': size_byte,
+                            'reservation': size_byte})
+        if proto == 'NFS':
+            export_loc = ("%(host)s:%(mountpoint)s/%(name)s" % arg)
+            manage_args.update({'sharenfs': 'sec=sys',
+                                'sharesmb': 'off'})
+        elif proto == 'CIFS':
+            export_loc = ("\\\\%(host)s\\%(name)s" % arg)
+            manage_args.update({'sharesmb': 'on',
+                                'sharenfs': 'off'})
+        else:
+            msg = _('Protocol %s is not supported.') % proto
+            LOG.error(msg)
+            raise exception.ManageInvalidShare(reason=msg)
+
+        self.zfssa.modify_share(lcfg.zfssa_pool, lcfg.zfssa_project,
+                                name, manage_args)
+        return {'size': size_gb, 'export_locations': export_loc}
+
+    def _verify_share_to_manage(self, name, details):
+        lcfg = self.configuration
+
+        if lcfg.zfssa_manage_policy == 'loose':
+            return
+
+        if 'custom:manila_managed' not in details:
+            msg = (_("Unknown if the share: %s to be managed is "
+                     "already being managed by Manila. Aborting manage "
+                     "share. Please add 'manila_managed' custom schema "
+                     "property to the share and set its value to False."
+                     "Alternatively, set Manila config property "
+                     "'zfssa_manage_policy' to 'loose' to remove this "
+                     "restriction.") % name)
+            LOG.error(msg)
+            raise exception.ManageInvalidShare(reason=msg)
+
+        if details['custom:manila_managed'] is True:
+            msg = (_("Share %s is already being managed by Manila.") % name)
+            LOG.error(msg)
+            raise exception.ManageInvalidShare(reason=msg)
+
+    def unmanage(self, share):
+        """Removes the specified share from Manila management.
+
+        This task involves only changing the custom:manila_managed
+        property to False. Current accesses to the share will be removed in
+        ZFSSA, as these accesses are removed in Manila.
+        """
+        name = share['id']
+        lcfg = self.configuration
+        managed = 'custom:manila_managed'
+        details = self._get_share_details(name)
+
+        if (managed not in details) or (details[managed] is not True):
+            msg = (_("Share %s is not being managed by the current Manila "
+                     "instance.") % name)
+            LOG.error(msg)
+            raise exception.UnmanageInvalidShare(reason=msg)
+
+        arg = {'custom:manila_managed': False}
+        if share['share_proto'] == 'NFS':
+            arg.update({'sharenfs': 'off'})
+        elif share['share_proto'] == 'CIFS':
+            arg.update({'sharesmb': 'off'})
+        else:
+            msg = (_("ZFSSA does not support %s protocol.") %
+                   share['share_proto'])
+            LOG.error(msg)
+            raise exception.UnmanageInvalidShare(reason=msg)
+        self.zfssa.modify_share(lcfg.zfssa_pool, lcfg.zfssa_project, name, arg)
+
+    def _get_share_details(self, name):
         lcfg = self.configuration
         details = self.zfssa.get_share(lcfg.zfssa_pool,
                                        lcfg.zfssa_project,
-                                       share['id'])
+                                       name)
         if not details:
-            msg = (_("Share %s doesn't exists.") % share['id'])
+            msg = (_("Share %s doesn't exist in ZFSSA.") % name)
-            raise exception.ManilaException(msg)
+            LOG.error(msg)
+            raise exception.ShareResourceNotFound(share_id=name)
+        return details
+
+    def ensure_share(self, context, share, share_server=None):
+        self._get_share_details(share['id'])
 
     def shrink_share(self, share, new_size, share_server=None):
         """Shrink a share to new_size."""

manila/tests/fake_zfssa.py

@@ -75,6 +75,9 @@ class FakeZFSSA(object):
     def get_project_stats(self, pool, project):
         pass
 
+    def create_schema(self, schema):
+        pass
+
 
 class FakeRestClient(object):
     """Fake ZFSSA Rest Client."""

manila/tests/share/drivers/zfssa/test_zfssarest.py

@@ -40,6 +40,12 @@ class ZFSSAApiTestCase(test.TestCase):
         self._zfssa = zfssarest.ZFSSAApi()
         self._zfssa.set_host('fakehost')
 
+        self.schema = {
+            'property': 'manila_managed',
+            'description': 'Managed by Manila',
+            'type': 'Boolean',
+        }
+
     def _create_response(self, status):
         response = fake_zfssa.FakeResponse(status)
         return response
@@ -390,3 +396,37 @@ class ZFSSAApiTestCase(test.TestCase):
                                                     self.project,
                                                     self.share,
                                                     data1)
+
+    def test_create_schema_negative(self):
+        self.mock_object(self._zfssa.rclient, 'get')
+        self.mock_object(self._zfssa.rclient, 'post')
+        self._zfssa.rclient.post.return_value = self._create_response(
+            restclient.Status.NOT_FOUND)
+
+        self.assertRaises(exception.ShareBackendException,
+                          self._zfssa.create_schema,
+                          self.schema)
+
+    def test_create_schema_property_exists(self):
+        self.mock_object(self._zfssa.rclient, 'get')
+        self.mock_object(self._zfssa.rclient, 'post')
+        self._zfssa.rclient.get.return_value = self._create_response(
+            restclient.Status.OK)
+
+        self._zfssa.create_schema(self.schema)
+
+        self.assertEqual(1, self._zfssa.rclient.get.call_count)
+        self.assertEqual(0, self._zfssa.rclient.post.call_count)
+
+    def test_create_schema(self):
+        self.mock_object(self._zfssa.rclient, 'get')
+        self.mock_object(self._zfssa.rclient, 'post')
+        self._zfssa.rclient.get.return_value = self._create_response(
+            restclient.Status.NOT_FOUND)
+        self._zfssa.rclient.post.return_value = self._create_response(
+            restclient.Status.CREATED)
+
+        self._zfssa.create_schema(self.schema)
+
+        self.assertEqual(1, self._zfssa.rclient.get.call_count)
+        self.assertEqual(1, self._zfssa.rclient.post.call_count)

manila/tests/share/drivers/zfssa/test_zfssashare.py

@@ -37,15 +37,15 @@ class ZFSSAShareDriverTestCase(test.TestCase):
         'name': 'fakename',
         'size': 1,
         'share_proto': 'NFS',
-        'export_location': '127.0.0.1:/mnt/nfs/volume-00002',
+        'export_location': '/mnt/nfs/volume-00002',
     }
 
     share2 = {
         'id': 'fakeid2',
         'name': 'fakename2',
         'size': 4,
-        'share_proto': 'NFS',
+        'share_proto': 'CIFS',
-        'export_location': '127.0.0.1:/mnt/nfs/volume-00003',
+        'export_location': '/mnt/nfs/volume-00003',
         'space_data': 3006477107
     }
 
@@ -78,6 +78,26 @@ class ZFSSAShareDriverTestCase(test.TestCase):
         self._driver = zfssashare.ZFSSAShareDriver(False, configuration=lcfg)
         self._driver.do_setup(self._context)
 
+        self.fake_proto_share = {
+            'id': self.share['id'],
+            'share_proto': 'fake_proto',
+            'export_locations': [{'path': self.share['export_location']}],
+        }
+
+        self.test_share = {
+            'id': self.share['id'],
+            'share_proto': 'NFS',
+            'export_locations': [{'path': self.share['export_location']}],
+        }
+
+        self.test_share2 = {
+            'id': self.share2['id'],
+            'share_proto': 'CIFS',
+            'export_locations': [{'path': self.share2['export_location']}],
+        }
+
+        self.driver_options = {'zfssa_name': self.share['name']}
+
     def _create_fake_config(self):
         def _safe_get(opt):
             return getattr(self.configuration, opt)
@@ -102,6 +122,7 @@ class ZFSSAShareDriverTestCase(test.TestCase):
         self.configuration.admin_network_config_group = (
             'fake_admin_network_config_group')
         self.configuration.driver_handles_share_servers = False
+        self.configuration.zfssa_manage_policy = 'strict'
 
     def test_create_share(self):
         self.mock_object(self._driver.zfssa, 'create_share')
@@ -283,3 +304,223 @@ class ZFSSAShareDriverTestCase(test.TestCase):
             lcfg.zfssa_project,
             self.share2['id'],
             arg)
+
+    def test_manage_invalid_option(self):
+        self.mock_object(self._driver, '_get_share_details')
+
+        # zfssa_name not in driver_options:
+        self.assertRaises(exception.ShareBackendException,
+                          self._driver.manage_existing,
+                          self.share,
+                          {})
+
+    def test_manage_no_share_details(self):
+        self.mock_object(self._driver, '_get_share_details')
+        self._driver._get_share_details.side_effect = (
+            exception.ShareResourceNotFound(share_id=self.share['name']))
+
+        self.assertRaises(exception.ShareResourceNotFound,
+                          self._driver.manage_existing,
+                          self.share,
+                          self.driver_options)
+
+    def test_manage_invalid_size(self):
+        details = {
+            'quota': 10,  # 10 bytes
+            'reservation': 10,
+        }
+        self.mock_object(self._driver, '_get_share_details')
+        self._driver._get_share_details.return_value = details
+
+        self.mock_object(self._driver.zfssa, 'get_project_stats')
+        self._driver.zfssa.get_project_stats.return_value = 900
+
+        # Share size is less than 1GB, but there is not enough free space
+        self.assertRaises(exception.ManageInvalidShare,
+                          self._driver.manage_existing,
+                          self.test_share,
+                          self.driver_options)
+
+    def test_manage_invalid_protocol(self):
+        self.mock_object(self._driver, '_get_share_details')
+        self._driver._get_share_details.return_value = {
+            'quota': self.share['size'] * units.Gi,
+            'reservation': self.share['size'] * units.Gi,
+            'custom:manila_managed': False,
+        }
+
+        self.assertRaises(exception.ManageInvalidShare,
+                          self._driver.manage_existing,
+                          self.fake_proto_share,
+                          self.driver_options)
+
+    def test_manage_unmanage_no_schema(self):
+        self.mock_object(self._driver, '_get_share_details')
+        self._driver._get_share_details.return_value = {}
+
+        # Share does not have custom:manila_managed property
+        # Test manage_existing():
+        self.assertRaises(exception.ManageInvalidShare,
+                          self._driver.manage_existing,
+                          self.test_share,
+                          self.driver_options)
+
+        # Test unmanage():
+        self.assertRaises(exception.UnmanageInvalidShare,
+                          self._driver.unmanage,
+                          self.test_share)
+
+    def test_manage_round_up_size(self):
+        details = {
+            'quota': 100,
+            'reservation': 50,
+            'custom:manila_managed': False,
+        }
+        self.mock_object(self._driver, '_get_share_details')
+        self._driver._get_share_details.return_value = details
+
+        self.mock_object(self._driver.zfssa, 'get_project_stats')
+        self._driver.zfssa.get_project_stats.return_value = 1 * units.Gi
+
+        ret = self._driver.manage_existing(self.test_share,
+                                           self.driver_options)
+
+        # Expect share size is 1GB
+        self.assertEqual(1, ret['size'])
+
+    def test_manage_not_enough_space(self):
+        details = {
+            'quota': 3.5 * units.Gi,
+            'reservation': 3.5 * units.Gi,
+            'custom:manila_managed': False,
+        }
+        self.mock_object(self._driver, '_get_share_details')
+        self._driver._get_share_details.return_value = details
+
+        self.mock_object(self._driver.zfssa, 'get_project_stats')
+        self._driver.zfssa.get_project_stats.return_value = 0.1 * units.Gi
+
+        self.assertRaises(exception.ManageInvalidShare,
+                          self._driver.manage_existing,
+                          self.test_share,
+                          self.driver_options)
+
+    def test_manage_unmanage_NFS(self):
+        lcfg = self.configuration
+        details = {
+            # Share size is 1GB
+            'quota': self.share['size'] * units.Gi,
+            'reservation': self.share['size'] * units.Gi,
+            'custom:manila_managed': False,
+        }
+        arg = {
+            'host': lcfg.zfssa_data_ip,
+            'mountpoint': self.share['export_location'],
+            'name': self.share['id'],
+        }
+        export_loc = "%(host)s:%(mountpoint)s/%(name)s" % arg
+        self.mock_object(self._driver, '_get_share_details')
+        self._driver._get_share_details.return_value = details
+
+        ret = self._driver.manage_existing(self.test_share,
+                                           self.driver_options)
+
+        self.assertEqual(export_loc, ret['export_locations'])
+        self.assertEqual(1, ret['size'])
+
+    def test_manage_unmanage_CIFS(self):
+        lcfg = self.configuration
+        details = {
+            # Share size is 1GB
+            'quota': self.share2['size'] * units.Gi,
+            'reservation': self.share2['size'] * units.Gi,
+            'custom:manila_managed': False,
+        }
+        arg = {
+            'host': lcfg.zfssa_data_ip,
+            'name': self.share2['id'],
+        }
+        export_loc = "\\\\%(host)s\\%(name)s" % arg
+        self.mock_object(self._driver, '_get_share_details')
+        self._driver._get_share_details.return_value = details
+
+        ret = self._driver.manage_existing(self.test_share2,
+                                           self.driver_options)
+
+        self.assertEqual(export_loc, ret['export_locations'])
+        self.assertEqual(4, ret['size'])
+
+    def test_unmanage_NFS(self):
+        self.mock_object(self._driver.zfssa, 'modify_share')
+        lcfg = self.configuration
+        details = {
+            'quota': self.share['size'] * units.Gi,
+            'reservation': self.share['size'] * units.Gi,
+            'custom:manila_managed': True,
+        }
+
+        arg = {
+            'custom:manila_managed': False,
+            'sharenfs': 'off',
+        }
+
+        self.mock_object(self._driver, '_get_share_details')
+        self._driver._get_share_details.return_value = details
+
+        self._driver.unmanage(self.test_share)
+
+        self._driver.zfssa.modify_share.assert_called_with(
+            lcfg.zfssa_pool,
+            lcfg.zfssa_project,
+            self.test_share['id'],
+            arg)
+
+    def test_unmanage_CIFS(self):
+        self.mock_object(self._driver.zfssa, 'modify_share')
+        lcfg = self.configuration
+        details = {
+            'quota': self.share2['size'] * units.Gi,
+            'reservation': self.share2['size'] * units.Gi,
+            'custom:manila_managed': True,
+        }
+
+        arg = {
+            'custom:manila_managed': False,
+            'sharesmb': 'off',
+        }
+
+        self.mock_object(self._driver, '_get_share_details')
+        self._driver._get_share_details.return_value = details
+
+        self._driver.unmanage(self.test_share2)
+
+        self._driver.zfssa.modify_share.assert_called_with(
+            lcfg.zfssa_pool,
+            lcfg.zfssa_project,
+            self.test_share2['id'],
+            arg)
+
+    def test_verify_share_to_manage_loose_policy(self):
+        # Temporarily change policy to loose
+        self.configuration.zfssa_manage_policy = 'loose'
+
+        ret = self._driver._verify_share_to_manage('sharename', {})
+
+        self.assertEqual(ret, None)
+        # Change it back to strict
+        self.configuration.zfssa_manage_policy = 'strict'
+
+    def test_verify_share_to_manage_no_property(self):
+        self.configuration.zfssa_manage_policy = 'strict'
+        self.assertRaises(exception.ManageInvalidShare,
+                          self._driver._verify_share_to_manage,
+                          'sharename',
+                          {})
+
+    def test_verify_share_to_manage_alredy_managed(self):
+        details = {'custom:manila_managed': True}
+
+        self.assertRaises(exception.ManageInvalidShare,
+                          self._driver._verify_share_to_manage,
+                          'sharename',
+                          details)

releasenotes/notes/zfssa-driver-add-share-manage-unmanage-9bd6d2e25cc86c35.yaml

@@ -0,0 +1,5 @@
+---
+features:
+ - Oracle ZFSSA driver now supports share manage/unmanage feature, where a
+   ZFSSA share can be brought under Manila's management, or can be released
+   from Manila's management.