Merge "add IPv6 support for CephFS/NFS back end"

.zuul.yaml

@@ -307,10 +307,13 @@
       - openstack/devstack-plugin-ceph
       - openstack/manila
       - openstack/manila-tempest-plugin
+      - openstack/neutron-dynamic-routing
+      # TODO(gouthamr): Remove the line below when neutron-dynamic-routing
+      # separates its tempest plugin from its tree
+      - openstack/neutron-tempest-plugin
       - openstack/python-manilaclient
       - openstack/tempest
 
-
 - job:
     name: manila-tempest-minimal-dsvm-dummy
     parent: manila-tempest-base

devstack/plugin.sh

@@ -1061,6 +1061,13 @@ function setup_ipv6 {
         iniset $MANILA_CONF DEFAULT data_node_access_ip $public_gateway_ipv4
     fi
 
+    if [ "$SHARE_DRIVER" == "manila.share.drivers.cephfs.driver.CephFSDriver" ]; then
+        for backend_name in ${MANILA_ENABLED_BACKENDS//,/ }; do
+            iniset $MANILA_CONF $backend_name cephfs_ganesha_export_ips $public_gateway_ipv4,$public_gateway_ipv6
+        done
+        iniset $MANILA_CONF DEFAULT data_node_access_ip $public_gateway_ipv4
+    fi
+
     # install Quagga for setting up the host routes dynamically
     install_package quagga
 

doc/source/admin/share_back_ends_feature_support_mapping.rst

@@ -146,7 +146,7 @@ Mapping of share drivers and share access rules support
 +----------------------------------------+--------------+--------------+----------------+------------+--------------+--------------+--------------+----------------+------------+------------+
 |              Oracle ZFSSA              |  NFS,CIFS(K) |      \-      |       \-       |     \-     |      \-      |      \-      |      \-      |       \-       |     \-     |     \-     |
 +----------------------------------------+--------------+--------------+----------------+------------+--------------+--------------+--------------+----------------+------------+------------+
-|                 CephFS                 |    NFS (P)   |      \-      |       \-       |     \-     |  CEPHFS (M)  |    NFS (P)   |      \-      |       \-       |     \-     | CEPHFS (N) |
+|                 CephFS                 |    NFS (P)   |    NFS (T)   |       \-       |     \-     |  CEPHFS (M)  |    NFS (P)   |    NFS (T)   |       \-       |     \-     | CEPHFS (N) |
 +----------------------------------------+--------------+--------------+----------------+------------+--------------+--------------+--------------+----------------+------------+------------+
 |                 Tegile                 |    NFS (M)   |      \-      |NFS (M),CIFS (M)|     \-     |      \-      |    NFS (M)   |      \-      |NFS (M),CIFS (M)|     \-     |     \-     |
 +----------------------------------------+--------------+--------------+----------------+------------+--------------+--------------+--------------+----------------+------------+------------+

manila/share/drivers/cephfs/driver.py

@@ -14,12 +14,15 @@
 #    under the License.
 
 
+import ipaddress
 import socket
 import sys
 
 from oslo_config import cfg
+from oslo_config import types
 from oslo_log import log
 from oslo_utils import units
+import six
 
 from manila.common import constants
 from manila import exception
@@ -27,6 +30,7 @@ from manila.i18n import _
 from manila.share import driver
 from manila.share.drivers import ganesha
 from manila.share.drivers.ganesha import utils as ganesha_utils
+from manila.share.drivers import helpers as driver_helpers
 from manila.share import share_types
 
 try:
@@ -78,8 +82,8 @@ cephfs_opts = [
                 default=False,
                 help="Whether the NFS-Ganesha server is remote to the driver."
                 ),
-    cfg.StrOpt('cephfs_ganesha_server_ip',
-               help="The IP address of the NFS-Ganesha server."),
+    cfg.HostAddressOpt('cephfs_ganesha_server_ip',
+                       help="The IP address of the NFS-Ganesha server."),
     cfg.StrOpt('cephfs_ganesha_server_username',
                default='root',
                help="The username to authenticate as in the remote "
@@ -91,6 +95,11 @@ cephfs_opts = [
                help="The password to authenticate as the user in the remote "
                     "Ganesha server host. This is not required if "
                     "'cephfs_ganesha_path_to_private_key' is configured."),
+    cfg.ListOpt('cephfs_ganesha_export_ips',
+                default='',
+                help="List of IPs to export shares. If not supplied, "
+                     "then the value of 'cephfs_ganesha_server_ip' "
+                     "will be used to construct share export locations."),
     cfg.StrOpt('cephfs_volume_mode',
                default=DEFAULT_VOLUME_MODE,
                help="The read/write/execute permissions mode for CephFS "
@@ -110,7 +119,7 @@ def cephfs_share_path(share):
 
 
 class CephFSDriver(driver.ExecuteMixin, driver.GaneshaMixin,
-                   driver.ShareDriver,):
+                   driver.ShareDriver):
     """Driver for the Ceph Filesystem."""
 
     def __init__(self, *args, **kwargs):
@@ -130,6 +139,8 @@ class CephFSDriver(driver.ExecuteMixin, driver.GaneshaMixin,
             raise exception.BadConfigurationException(
                 msg % self.configuration.cephfs_volume_mode)
 
+        self.ipv6_implemented = True
+
     def do_setup(self, context):
         if self.configuration.cephfs_protocol_helper_type.upper() == "CEPHFS":
             protocol_helper_class = getattr(
@@ -145,6 +156,10 @@ class CephFSDriver(driver.ExecuteMixin, driver.GaneshaMixin,
 
         self.protocol_helper.init_helper()
 
+    def check_for_setup_error(self):
+        """Returns an error if prerequisites aren't met."""
+        self.protocol_helper.check_for_setup_error()
+
     def _update_share_stats(self):
         stats = self.volume_client.rados.get_cluster_stats()
 
@@ -174,7 +189,8 @@ class CephFSDriver(driver.ExecuteMixin, driver.GaneshaMixin,
             'snapshot_support': self.configuration.safe_get(
                 'cephfs_enable_snapshots'),
         }
-        super(CephFSDriver, self)._update_share_stats(data)
+        super(    # pylint: disable=no-member
+            CephFSDriver, self)._update_share_stats(data)
 
     def _to_bytes(self, gigs):
         """Convert a Manila size into bytes.
@@ -337,6 +353,9 @@ class CephFSDriver(driver.ExecuteMixin, driver.GaneshaMixin,
             self._volume_client.disconnect()
             self._volume_client = None
 
+    def get_configured_ip_versions(self):
+        return self.protocol_helper.get_configured_ip_versions()
+
 
 class NativeProtocolHelper(ganesha.NASHelperBase):
     """Helper class for native CephFS protocol"""
@@ -353,6 +372,10 @@ class NativeProtocolHelper(ganesha.NASHelperBase):
     def _init_helper(self):
         pass
 
+    def check_for_setup_error(self):
+        """Returns an error if prerequisites aren't met."""
+        return
+
     def get_export_locations(self, share, cephfs_volume):
         # To mount this you need to know the mon IPs and the path to the volume
         mon_addrs = self.volume_client.get_mon_addrs()
@@ -460,6 +483,9 @@ class NativeProtocolHelper(ganesha.NASHelperBase):
 
         return access_keys
 
+    def get_configured_ip_versions(self):
+        return [4]
+
 
 class NFSProtocolHelper(ganesha.GaneshaNASHelper2):
 
@@ -487,20 +513,40 @@ class NFSProtocolHelper(ganesha.GaneshaNASHelper2):
 
         if not hasattr(self, 'ceph_vol_client'):
             self.ceph_vol_client = kwargs.pop('ceph_vol_client')
+        self.export_ips = config_object.cephfs_ganesha_export_ips
+        if not self.export_ips:
+            self.export_ips = [self.ganesha_host]
+        self.configured_ip_versions = set()
+        self.config = config_object
+
+    def check_for_setup_error(self):
+        """Returns an error if prerequisites aren't met."""
+        host_address_obj = types.HostAddress()
+        for export_ip in self.config.cephfs_ganesha_export_ips:
+            try:
+                host_address_obj(export_ip)
+            except ValueError:
+                msg = (_("Invalid list member of 'cephfs_ganesha_export_ips' "
+                         "option supplied %s -- not a valid IP address or "
+                         "hostname.") % export_ip)
+                raise exception.InvalidParameterValue(err=msg)
 
     def get_export_locations(self, share, cephfs_volume):
-        export_location = "{server_address}:{path}".format(
-            server_address=self.ganesha_host,
-            path=cephfs_volume['mount_path'])
+        export_locations = []
+        for export_ip in self.export_ips:
+            export_path = "{server_address}:{mount_path}".format(
+                server_address=driver_helpers.escaped_address(export_ip),
+                mount_path=cephfs_volume['mount_path'])
 
-        LOG.info("Calculated export location for share %(id)s: %(loc)s",
-                 {"id": share['id'], "loc": export_location})
-
-        return {
-            'path': export_location,
-            'is_admin_only': False,
-            'metadata': {},
-        }
+            LOG.info("Calculated export path for share %(id)s: %(epath)s",
+                     {"id": share['id'], "epath": export_path})
+            export_location = {
+                'path': export_path,
+                'is_admin_only': False,
+                'metadata': {},
+            }
+            export_locations.append(export_location)
+        return export_locations
 
     def _default_config_hook(self):
         """Callback to provide default export block."""
@@ -540,3 +586,19 @@ class NFSProtocolHelper(ganesha.GaneshaNASHelper2):
         """Callback to provide pseudo path."""
         volume_path = cephfs_share_path(share)
         return self.ceph_vol_client._get_path(volume_path)
+
+    def get_configured_ip_versions(self):
+        if not self.configured_ip_versions:
+            try:
+                for export_ip in self.export_ips:
+                    self.configured_ip_versions.add(
+                        ipaddress.ip_address(six.text_type(export_ip)).version)
+            except Exception:
+                # export_ips contained a hostname, safest thing is to
+                # claim support for IPv4 and IPv6 address families
+                LOG.warning("Setting configured IP versions to [4, 6] since "
+                            "a hostname (rather than IP address) was supplied "
+                            "in 'cephfs_ganesha_server_ip' or "
+                            "in 'cephfs_ganesha_export_ips'.")
+                return [4, 6]
+        return list(self.configured_ip_versions)

manila/share/drivers/helpers.py

@@ -181,6 +181,14 @@ def nfs_synchronized(f):
     return wrapped_func
 
 
+def escaped_address(address):
+    addr = ipaddress.ip_address(six.text_type(address))
+    if addr.version == 4:
+        return six.text_type(addr)
+    else:
+        return '[%s]' % six.text_type(addr)
+
+
 class NFSHelper(NASHelperBase):
     """Interface to work with share."""
 
@@ -191,24 +199,16 @@ class NFSHelper(NASHelperBase):
         if 'public_addresses' in server_copy:
             for address in server_copy['public_addresses']:
                 public_addresses.append(
-                    self._escaped_address(address))
+                    escaped_address(address))
             server_copy['public_addresses'] = public_addresses
 
         for t in ['public_address', 'admin_ip', 'ip']:
             address = server_copy.get(t)
             if address is not None:
-                server_copy[t] = self._escaped_address(address)
+                server_copy[t] = escaped_address(address)
 
         return self.get_exports_for_share(server_copy, path)
 
-    @staticmethod
-    def _escaped_address(address):
-        addr = ipaddress.ip_address(six.text_type(address))
-        if addr.version == 4:
-            return six.text_type(addr)
-        else:
-            return '[%s]' % six.text_type(addr)
-
     def init_helper(self, server):
         try:
             self._ssh_exec(server, ['sudo', 'exportfs'])

manila/tests/share/drivers/cephfs/test_driver.py

@@ -136,6 +136,16 @@ class CephFSDriverTestCase(test.TestCase):
 
         self.assertEqual(DEFAULT_VOLUME_MODE, self._driver._cephfs_volume_mode)
 
+    @ddt.data('cephfs', 'nfs')
+    def test_check_for_setup_error(self, protocol_helper):
+        self._driver.configuration.cephfs_protocol_helper_type = (
+            protocol_helper)
+
+        self._driver.check_for_setup_error()
+
+        (self._driver.protocol_helper.check_for_setup_error.
+            assert_called_once_with())
+
     def test_create_share(self):
         cephfs_volume = {"mount_path": "/foo/bar"}
 
@@ -342,6 +352,7 @@ class CephFSDriverTestCase(test.TestCase):
         vc.connect.assert_called_once_with(premount_evict=None)
 
     def test_update_share_stats(self):
+        self._driver.get_configured_ip_versions = mock.Mock(return_value=[4])
         self._driver._volume_client
         self._driver._update_share_stats()
         result = self._driver._stats
@@ -359,6 +370,16 @@ class CephFSDriverTestCase(test.TestCase):
                           self._context,
                           self._share)
 
+    @ddt.data('cephfs', 'nfs')
+    def test_get_configured_ip_versions(self, protocol_helper):
+        self._driver.configuration.cephfs_protocol_helper_type = (
+            protocol_helper)
+
+        self._driver.get_configured_ip_versions()
+
+        (self._driver.protocol_helper.get_configured_ip_versions.
+            assert_called_once_with())
+
 
 @ddt.ddt
 class NativeProtocolHelperTestCase(test.TestCase):
@@ -379,6 +400,13 @@ class NativeProtocolHelperTestCase(test.TestCase):
             ceph_vol_client=MockVolumeClientModule.CephFSVolumeClient()
         )
 
+    def test_check_for_setup_error(self):
+        expected = None
+
+        result = self._native_protocol_helper.check_for_setup_error()
+
+        self.assertEqual(expected, result)
+
     def test_get_export_locations(self):
         vc = self._native_protocol_helper.volume_client
         fake_cephfs_volume = {'mount_path': '/foo/bar'}
@@ -534,6 +562,13 @@ class NativeProtocolHelperTestCase(test.TestCase):
             vc.authorize.assert_called_once_with(
                 driver.cephfs_share_path(self._share), "alice")
 
+    def test_get_configured_ip_versions(self):
+        expected = [4]
+
+        result = self._native_protocol_helper.get_configured_ip_versions()
+
+        self.assertEqual(expected, result)
+
 
 @ddt.ddt
 class NFSProtocolHelperTestCase(test.TestCase):
@@ -558,6 +593,38 @@ class NFSProtocolHelperTestCase(test.TestCase):
             self.fake_conf,
             ceph_vol_client=self._volume_client)
 
+    @ddt.data(
+        (['fakehost', 'some.host.name', 'some.host.name.', '1.1.1.0'], False),
+        (['fakehost', 'some.host.name', 'some.host.name.', '1.1..1.0'], True),
+        (['fakehost', 'some.host.name', 'some.host.name', '1.1.1.256'], True),
+        (['fakehost..', 'some.host.name', 'some.host.name', '1.1.1.0'], True),
+        (['fakehost', 'some.host.name..', 'some.host.name', '1.1.1.0'], True),
+        (['fakehost', 'some.host.name', 'some.host.name.', '1.1..1.0'], True),
+        (['fakehost', 'some.host.name', '1.1.1.0/24'], True),
+        (['fakehost', 'some.host.name', '1.1.1.0', '1001::1001'], False),
+        (['fakehost', 'some.host.name', '1.1.1.0', '1001:1001'], True),
+        (['fakehost', 'some.host.name', '1.1.1.0', '1001::1001:'], True),
+        (['fakehost', 'some.host.name', '1.1.1.0', '1001::1001.'], True),
+        (['fakehost', 'some.host.name', '1.1.1.0', '1001::1001/129.'], True),
+    )
+    @ddt.unpack
+    def test_check_for_setup_error(self, cephfs_ganesha_export_ips, raises):
+        fake_conf = configuration.Configuration(None)
+        fake_conf.set_default('cephfs_ganesha_export_ips',
+                              cephfs_ganesha_export_ips)
+
+        helper = driver.NFSProtocolHelper(
+            self._execute,
+            fake_conf,
+            ceph_vol_client=MockVolumeClientModule.CephFSVolumeClient()
+        )
+
+        if raises:
+            self.assertRaises(exception.InvalidParameterValue,
+                              helper.check_for_setup_error)
+        else:
+            self.assertIsNone(helper.check_for_setup_error())
+
     @ddt.data(False, True)
     def test_init_executor_type(self, ganesha_server_is_remote):
         fake_conf = configuration.Configuration(None)
@@ -612,17 +679,104 @@ class NFSProtocolHelperTestCase(test.TestCase):
             driver.socket.gethostname.assert_called_once_with()
             driver.LOG.info.assert_called_once()
 
-    def test_get_export_locations(self):
+    def test_get_export_locations_no_export_ips_configured(self):
         cephfs_volume = {"mount_path": "/foo/bar"}
+        fake_conf = configuration.Configuration(None)
+        fake_conf.set_default('cephfs_ganesha_server_ip', '1.2.3.4')
 
-        ret = self._nfs_helper.get_export_locations(self._share,
-                                                    cephfs_volume)
+        helper = driver.NFSProtocolHelper(
+            self._execute,
+            fake_conf,
+            ceph_vol_client=MockVolumeClientModule.CephFSVolumeClient()
+        )
+
+        ret = helper.get_export_locations(self._share,
+                                          cephfs_volume)
         self.assertEqual(
-            {
-                'path': 'fakeip:/foo/bar',
+            [{
+                'path': '1.2.3.4:/foo/bar',
                 'is_admin_only': False,
                 'metadata': {}
-            }, ret)
+            }], ret)
+
+    def test_get_export_locations_with_export_ips_configured(self):
+        fake_conf = configuration.Configuration(None)
+        conf_args_list = [
+            ('cephfs_ganesha_server_ip', '1.2.3.4'),
+            ('cephfs_ganesha_export_ips', '127.0.0.1,fd3f:c057:1192:1::1,::1')]
+        for args in conf_args_list:
+            fake_conf.set_default(*args)
+
+        helper = driver.NFSProtocolHelper(
+            self._execute,
+            fake_conf,
+            ceph_vol_client=MockVolumeClientModule.CephFSVolumeClient()
+        )
+
+        cephfs_volume = {"mount_path": "/foo/bar"}
+
+        ret = helper.get_export_locations(self._share, cephfs_volume)
+
+        self.assertEqual(
+            [
+                {
+                    'path': '127.0.0.1:/foo/bar',
+                    'is_admin_only': False,
+                    'metadata': {},
+                },
+                {
+                    'path': '[fd3f:c057:1192:1::1]:/foo/bar',
+                    'is_admin_only': False,
+                    'metadata': {},
+                },
+                {
+                    'path': '[::1]:/foo/bar',
+                    'is_admin_only': False,
+                    'metadata': {},
+                },
+            ], ret)
+
+    @ddt.data(('some.host.name', None, [4, 6]), ('host.', None, [4, 6]),
+              ('1001::1001', None, [6]), ('1.1.1.0', None, [4]),
+              (None, ['1001::1001', '1.1.1.0'], [6, 4]),
+              (None, ['1001::1001'], [6]), (None, ['1.1.1.0'], [4]),
+              (None, ['1001::1001/129', '1.1.1.0'], [4, 6]))
+    @ddt.unpack
+    def test_get_configured_ip_versions(
+            self, cephfs_ganesha_server_ip, cephfs_ganesha_export_ips,
+            configured_ip_version):
+        fake_conf = configuration.Configuration(None)
+        conf_args_list = [
+            ('cephfs_ganesha_server_ip', cephfs_ganesha_server_ip),
+            ('cephfs_ganesha_export_ips', cephfs_ganesha_export_ips)]
+
+        for args in conf_args_list:
+            fake_conf.set_default(*args)
+
+        helper = driver.NFSProtocolHelper(
+            self._execute,
+            fake_conf,
+            ceph_vol_client=MockVolumeClientModule.CephFSVolumeClient()
+        )
+
+        self.assertEqual(set(configured_ip_version),
+                         set(helper.get_configured_ip_versions()))
+
+    def test_get_configured_ip_versions_already_set(self):
+        fake_conf = configuration.Configuration(None)
+        helper = driver.NFSProtocolHelper(
+            self._execute,
+            fake_conf,
+            ceph_vol_client=MockVolumeClientModule.CephFSVolumeClient()
+        )
+
+        ip_versions = ['foo', 'bar']
+
+        helper.configured_ip_versions = ip_versions
+
+        result = helper.get_configured_ip_versions()
+
+        self.assertEqual(ip_versions, result)
 
     def test_default_config_hook(self):
         fake_conf_dict = {'key': 'value1'}

playbooks/legacy/manila-tempest-minimal-dsvm-cephfs-nfs/run.yaml

@@ -30,7 +30,9 @@
           cat << 'EOF' >>"/tmp/dg-local.conf"
           [[local|localrc]]
           enable_plugin manila https://opendev.org/openstack/manila
-
+          enable_plugin manila-tempest-plugin https://opendev.org/openstack/manila-tempest-plugin
+          enable_plugin neutron-dynamic-routing https://opendev.org/openstack/neutron-dynamic-routing
+          enable_plugin neutron-tempest-plugin https://opendev.org/openstack/neutron-tempest-plugin
           enable_plugin devstack-plugin-ceph https://opendev.org/openstack/devstack-plugin-ceph
 
           # Enable CephFS as the backend for Manila.
@@ -62,12 +64,12 @@
           set -x
           export PYTHONUNBUFFERED=true
           export DEVSTACK_GATE_NEUTRON=1
-          export ENABLED_SERVICES=tempest
-          export PROJECTS="openstack/devstack-plugin-ceph $PROJECTS"
           export DEVSTACK_PROJECT_FROM_GIT="python-manilaclient"
-          export KEEP_LOCALRC=1
-          export PROJECTS="openstack/manila-tempest-plugin $PROJECTS"
 
+          export MANILA_SETUP_IPV6=True
+          export RUN_MANILA_IPV6_TESTS=True
+
+          # Basic services needed for minimal job
           OVERRIDE_ENABLED_SERVICES=key,mysql,rabbit,tempest
           # Enable glance for scenario tests
           OVERRIDE_ENABLED_SERVICES+=,g-api,g-reg
@@ -75,6 +77,18 @@
           OVERRIDE_ENABLED_SERVICES+=,n-api,n-cpu,n-cond,n-sch,n-crt,n-cauth,n-obj
           # Enable neutron for scenario tests
           OVERRIDE_ENABLED_SERVICES+=,q-svc,q-dhcp,q-meta,q-l3,q-agt
+          # Enable tls-proxy
+          OVERRIDE_ENABLED_SERVICES+=,tls-proxy
+          OVERRIDE_ENABLED_SERVICES+=,placement-api,placement-client
+          export OVERRIDE_ENABLED_SERVICES
+
+          # Keep localrc to be able to set some vars in pre_test_hook
+          export KEEP_LOCALRC=1
+          PROJECTS="openstack/devstack-plugin-ceph $PROJECTS"
+          PROJECTS="openstack/manila-tempest-plugin $PROJECTS"
+          PROJECTS="openstack/neutron-dynamic-routing $PROJECTS"
+          PROJECTS="openstack/neutron-tempest-plugin $PROJECTS"
+          export PROJECTS
 
           export DEVSTACK_GATE_USE_PYTHON3=True
 

releasenotes/notes/cephfs-nfs-ipv6-support-2ffd9c0448c2f47e.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - IPv6 support for CephFS Manila driver with NFS gateway.
+