RBD: Run flatten in a different thread when cloning a volume

The current implementation of create_cloned_volume calls flatten
directly, and this makes whole thread of cinder-volume blocked by that
flatten call. This causes heartbeat timeout in RabbitMQ when cloning
a volume with rbd backend.

This patch makes sure that flatten is executed in a different thread,
to allow heatbeat thread to run while flattening a rbd image.

Closes-Bug: #1898918
Change-Id: I9f28260008117abcebfc96dbe69bf892f5cd14fe

cinder/tests/unit/volume/drivers/test_rbd.py

@@ -1163,6 +1163,9 @@ class RBDTestCase(test.TestCase):
     def test_create_cloned_volume_w_flatten(self, mock_enable_repl):
         self.cfg.rbd_max_clone_depth = 1
 
+        client = self.mock_client.return_value
+        client.__enter__.return_value = client
+
         with mock.patch.object(self.driver, '_get_clone_info') as \
                 mock_get_clone_info:
             mock_get_clone_info.return_value = (
@@ -1192,9 +1195,13 @@ class RBDTestCase(test.TestCase):
                  .assert_called_once_with('.'.join(
                      (self.volume_b.name, 'clone_snap'))))
 
-                # We expect the driver to close both volumes, so 2 is expected
+                self.mock_proxy.assert_called_once_with(
+                    self.driver, self.volume_b.name,
+                    client=client, ioctx=client.ioctx)
+
+                # Source volume is closed by direct call of close()
                 self.assertEqual(
-                    2, self.mock_rbd.Image.return_value.close.call_count)
+                    1, self.mock_rbd.Image.return_value.close.call_count)
                 self.assertTrue(mock_get_clone_depth.called)
                 mock_enable_repl.assert_not_called()
 

cinder/volume/drivers/rbd.py

@@ -704,11 +704,13 @@ class RBDDriver(driver.CloneableImageVD, driver.MigrateVD,
                 LOG.info("maximum clone depth (%d) has been reached - "
                          "flattening dest volume",
                          self.configuration.rbd_max_clone_depth)
-                dest_volume = self.rbd.Image(client.ioctx, dest_name)
+
+                # Flatten destination volume
                 try:
-                    # Flatten destination volume
-                    LOG.debug("flattening dest volume %s", dest_name)
-                    dest_volume.flatten()
+                    with RBDVolumeProxy(self, dest_name, client=client,
+                                        ioctx=client.ioctx) as dest_volume:
+                        LOG.debug("flattening dest volume %s", dest_name)
+                        dest_volume.flatten()
                 except Exception as e:
                     msg = (_("Failed to flatten volume %(volume)s with "
                              "error: %(error)s.") %
@@ -717,8 +719,6 @@ class RBDDriver(driver.CloneableImageVD, driver.MigrateVD,
                     LOG.exception(msg)
                     src_volume.close()
                     raise exception.VolumeBackendAPIException(data=msg)
-                finally:
-                    dest_volume.close()
 
                 try:
                     # remove temporary snap

releasenotes/notes/bug-1898918-b24a93d7d5aff238.yaml

@@ -0,0 +1,7 @@
+---
+fixes:
+  - |
+    RBD driver `Bug #1898918
+    <https://bugs.launchpad.net/cinder/+bug/1898918>`_: Fix thread block caused
+    by the flatten operation during cloning a volume. Now the flatten operation
+    is executed in a different thread.