Remove rbd-nbd and improve rbd-ko

We prematurely added the rbd-nbd feature, so we are removing it for the
time being.

We also improve rbd-ko to be able to run with the ceph-common package
only installed on the container, so we no longer need to have it
installed on the host.

Dockerfile

@@ -8,7 +8,7 @@ LABEL maintainers="Gorka Eguileor <geguileo@redhat.com>" \
 
 RUN yum -y install targetcli iscsi-initiator-utils device-mapper-multipath epel-release lvm2 which && \
     yum -y install python2-pip python-devel gcc && \
-    yum -y install python-rbd ceph-common rbd-nbd git && \
+    yum -y install python-rbd ceph-common git && \
     # Need new setuptools version or we'll get "SyntaxError: '<' operator not allowed in environment markers" when installing Cinder
     pip install 'setuptools>=38.6.0' && \
     git clone 'https://github.com/openstack/cinder.git' && \

Dockerfile-latest

@@ -9,7 +9,7 @@ LABEL maintainers="Gorka Eguileor <geguileo@redhat.com>" \
 RUN yum -y install targetcli iscsi-initiator-utils device-mapper-multipath epel-release lvm2 which && \
     echo yum -y install python2-pip centos-release-openstack-${RELEASE} > /root/whatever && \
     yum -y install python2-pip centos-release-openstack-${RELEASE} && \
-    yum -y install openstack-cinder python-rbd ceph-common rbd-nbd && \
+    yum -y install openstack-cinder python-rbd ceph-common && \
     pip install --no-cache-dir 'krest>=1.3.0' 'purestorage>=1.6.0' && \
     yum clean all && \
     rm -rf /var/cache/yum

Dockerfile-release

@@ -8,7 +8,7 @@ LABEL maintainers="Gorka Eguileor <geguileo@redhat.com>" \
 
 RUN yum -y install targetcli iscsi-initiator-utils device-mapper-multipath epel-release lvm2 which && \
     yum -y install python2-pip centos-release-openstack-$RELEASE && \
-    yum -y install openstack-cinder python-rbd ceph-common rbd-nbd && \
+    yum -y install openstack-cinder python-rbd ceph-common && \
     yum clean all && \
     rm -rf /var/cache/yum && \
     pip install --no-cache-dir --process-dependency-links cinderlib 'krest>=1.3.0' 'purestorage>=1.6.0'

HISTORY.rst

@@ -2,6 +2,17 @@
 History
 =======
 
+0.3.4 (2019-01-26)
+------------------
+
+- Features:
+
+  - RBD volumes in container without RBD installed on host
+
+- Removals:
+
+  - RBD-NBD support was prematurely added, removed in this release
+
 0.3.3 (2019-01-24)
 ------------------
 

cinderlib/nos_brick.py

@@ -50,11 +50,9 @@ class RBDConnector(connectors.rbd.RBDConnector):
 
     We need a third one, local attachment on non controller node.
     """
-    rbd_nbd_installed = True
-
     def connect_volume(self, connection_properties):
         # NOTE(e0ne): sanity check if ceph-common is installed.
-        self._setup_rbd_methods()
+        self._setup_rbd_class()
 
         # Extract connection parameters and generate config file
         try:
@@ -71,48 +69,51 @@ class RBDConnector(connectors.rbd.RBDConnector):
         conf = self._create_ceph_conf(monitor_ips, monitor_ports,
                                       str(cluster_name), user,
                                       keyring)
-        dev_path = self._connect_volume(pool, volume, conf,
-                                        connection_properties)
 
-        return {'path': dev_path,
+        link_name = self.get_rbd_device_name(pool, volume)
+        real_path = os.path.realpath(link_name)
+
+        try:
+            # Map RBD volume if it's not already mapped
+            if not os.path.islink(link_name) or not os.path.exists(real_path):
+                cmd = ['rbd', 'map', volume, '--pool', pool, '--conf', conf]
+                cmd += self._get_rbd_args(connection_properties)
+                stdout, stderr = self._execute(*cmd,
+                                               root_helper=self._root_helper,
+                                               run_as_root=True)
+                real_path = stdout.strip()
+                # The host may not have RBD installed, and therefore won't
+                # create the symlinks, ensure they exist
+                if self.containerized:
+                    self._ensure_link(real_path, link_name)
+        except Exception:
+            fileutils.delete_if_exists(conf)
+
+        return {'path': real_path,
                 'conf': conf,
                 'type': 'block'}
 
-    def _rbd_connect_volume(self, pool, volume, conf, connection_properties):
+    def _ensure_link(self, source, link_name):
-        # Map RBD volume if it's not already mapped
+        self._ensure_dir(os.path.dirname(link_name))
-        dev_path = self.get_rbd_device_name(pool, volume)
+        if self.im_root:
-        if (not os.path.islink(dev_path) or
+            try:
-                not os.path.exists(os.path.realpath(dev_path))):
+                os.symlink(source, link_name)
-            cmd = ['rbd', 'map', volume, '--pool', pool, '--conf', conf]
+            except Exception:
-            cmd += self._get_rbd_args(connection_properties)
+                pass
-            self._execute(*cmd, root_helper=self._root_helper,
+        else:
+            self._execute('ln', '-s', '-f', source, link_name,
                           run_as_root=True)
-        return os.path.realpath(dev_path)
-
-    def _get_nbd_device_name(self, pool, volume, conf, connection_properties):
-        cmd = ('rbd-nbd', 'list-mapped', '--conf', conf)
-        cmd += self._get_rbd_args(connection_properties)
-        stdout, stderr = self._execute(*cmd, root_helper=self._root_helper,
-                                       run_as_root=True)
-        for line in stdout.strip().splitlines():
-            pid, dev_pool, image, snap, device = line.split(None)
-            if dev_pool == pool and image == volume:
-                return device
-        return None
-
-    def _nbd_connect_volume(self, pool, volume, conf, connection_properties):
-        dev_path = self._get_nbd_device_name(pool, volume, conf,
-                                             connection_properties)
-        if not dev_path:
-            cmd = ['rbd-nbd', 'map', volume, '--conf', conf]
-            cmd += self._get_rbd_args(connection_properties)
-            dev_path, stderr = self._execute(*cmd,
-                                             root_helper=self._root_helper,
-                                             run_as_root=True)
-        return dev_path.strip()
 
     def check_valid_device(self, path, run_as_root=True):
         """Verify an existing RBD handle is connected and valid."""
+        if self.im_root:
+            try:
+                with open(path, 'r') as f:
+                    f.read(4096)
+            except Exception:
+                return False
+            return True
+
         try:
             self._execute('dd', 'if=' + path, 'of=/dev/null', 'bs=4096',
                           'count=1', root_helper=self._root_helper,
@@ -123,47 +124,43 @@ class RBDConnector(connectors.rbd.RBDConnector):
 
     def disconnect_volume(self, connection_properties, device_info,
                           force=False, ignore_errors=False):
-
+        self._setup_rbd_class()
         pool, volume = connection_properties['name'].split('/')
         conf_file = device_info['conf']
-        if self.rbd_nbd_installed:
+        link_name = self.get_rbd_device_name(pool, volume)
-            dev_path = self._get_nbd_device_name(pool, volume, conf_file,
+        real_dev_path = os.path.realpath(link_name)
-                                                 connection_properties)
-            executable = 'rbd-nbd'
-        else:
-            dev_path = self.get_rbd_device_name(pool, volume)
-            executable = 'rbd'
 
-        real_dev_path = os.path.realpath(dev_path)
         if os.path.exists(real_dev_path):
-            cmd = [executable, 'unmap', dev_path, '--conf', conf_file]
+            cmd = ['rbd', 'unmap', real_dev_path, '--conf', conf_file]
             cmd += self._get_rbd_args(connection_properties)
             self._execute(*cmd, root_helper=self._root_helper,
                           run_as_root=True)
+
+            if self.containerized:
+                unlink_root(link_name)
         fileutils.delete_if_exists(conf_file)
 
-    def _check_installed(self):
+    def _ensure_dir(self, path):
+        if self.im_root:
+            os.makedirs(path)
+        else:
+            self._execute('mkdir', '-p', path, run_as_root=True)
+
+    def _setup_class(self):
         try:
             self._execute('which', 'rbd')
         except putils.ProcessExecutionError:
             msg = 'ceph-common package not installed'
             raise exception.BrickException(msg)
 
-        try:
+        RBDConnector.im_root = os.getuid() == 0
-            self._execute('which', 'rbd-nbd')
+        # Check if we are running containerized
-            RBDConnector._connect_volume = RBDConnector._nbd_connect_volume
+        RBDConnector.containerized = os.stat('/proc').st_dev > 4
-            RBDConnector._get_rbd_args = RBDConnector._get_nbd_args
-        except putils.ProcessExecutionError:
-            RBDConnector.rbd_nbd_installed = False
 
         # Don't check again to speed things on following connections
-        RBDConnector.setup_rbd_methods = lambda *args: None
+        RBDConnector._setup_rbd_class = lambda *args: None
 
-    def _get_nbd_args(self, connection_properties):
+    _setup_rbd_class = _setup_class
-        return ('--id', connection_properties['auth_username'])
-
-    _setup_rbd_methods = _check_installed
-    _connect_volume = _rbd_connect_volume
 
 
 ROOT_HELPER = 'sudo'
@@ -174,10 +171,17 @@ def unlink_root(*links, **kwargs):
     raise_at_end = kwargs.get('raise_at_end', False)
     exc = exception.ExceptionChainer()
     catch_exception = no_errors or raise_at_end
-    for link in links:
+
-        with exc.context(catch_exception, 'Unlink failed for %s', link):
+    error_msg = 'Some unlinks failed for %s'
-            putils.execute('unlink', link, run_as_root=True,
+    if os.getuid() == 0:
+        for link in links:
+            with exc.context(catch_exception, error_msg, links):
+                os.unlink(link)
+    else:
+        with exc.context(catch_exception, error_msg, links):
+            putils.execute('rm', *links, run_as_root=True,
                            root_helper=ROOT_HELPER)
+
     if not no_errors and raise_at_end and exc:
         raise exc