openstack
/
cinder
Code Issues Proposed changes

RBD: Retry delete if VolumeIsBusy in _copy_image_to_volume 

Cinder can fail to create an image-based volume if RBD mirroring
is enabled. With the journaling-based approach to RBD mirroring,
ceph will still create a snapshot as a result of volume creation.
The volume create in _create_from_image_download() results in
a snapshot getting created, resulting in a race where delete_volume()
gets a VolumeIsBusy exception.

Change-Id: Ib80e04512ec34a390e9e17af2f3544e18cad8598
Closes-Bug: #1900775
(cherry picked from commit 6231d26667)
(cherry picked from commit e1ed30838c)
This commit is contained in:
Corey Bryant 2020-10-22 10:55:33 -04:00
parent d00b1de8a9
commit 4f40f059f8
2 changed files with 27 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
cinder/tests/unit/volume/drivers/test_rbd.py
View File

@ -1311,17 +1311,29 @@ class RBDTestCase(test.TestCase):
self.driver._is_cloneable(location, {'disk_format': f}))
self.assertTrue(mock_get_fsid.called)
def _copy_image(self):
def _copy_image(self, volume_busy=False):
with mock.patch.object(tempfile, 'NamedTemporaryFile'):
with mock.patch.object(os.path, 'exists') as mock_exists:
mock_exists.return_value = True
with mock.patch.object(image_utils, 'fetch_to_raw'):
with mock.patch.object(self.driver, 'delete_volume'):
with mock.patch.object(self.driver, 'delete_volume') \
as mock_dv:
with mock.patch.object(self.driver, '_resize'):
mock_image_service = mock.MagicMock()
args = [None, self.volume_a,
mock_image_service, None]
self.driver.copy_image_to_volume(*args)
if volume_busy:
mock_dv.side_effect = (
exception.VolumeIsBusy("doh"))
self.assertRaises(
exception.VolumeIsBusy,
self.driver.copy_image_to_volume,
*args)
self.assertEqual(
self.cfg.rados_connection_retries,
mock_dv.call_count)
else:
self.driver.copy_image_to_volume(*args)
@mock.patch('cinder.volume.drivers.rbd.fileutils.delete_if_exists')
@mock.patch('cinder.volume.volume_utils.check_encryption_provider',
@ -1371,6 +1383,11 @@ class RBDTestCase(test.TestCase):
self.cfg.image_conversion_dir = '/var/run/cinder/tmp'
self._copy_image_encrypted()
@common_mocks
def test_copy_image_busy_volume(self):
self.cfg.image_conversion_dir = '/var/run/cinder/tmp'
self._copy_image(volume_busy=True)
@ddt.data(True, False)
@common_mocks
@mock.patch('cinder.volume.drivers.rbd.RBDDriver._get_usage_info')

8
cinder/volume/drivers/rbd.py
View File

@ -1601,7 +1601,13 @@ class RBDDriver(driver.CloneableImageVD, driver.MigrateVD,
if encrypted:
self._encrypt_image(context, volume, tmp_dir, tmp.name)
self.delete_volume(volume)
@utils.retry(exception.VolumeIsBusy,
self.configuration.rados_connection_interval,
self.configuration.rados_connection_retries)
def _delete_volume(volume):
self.delete_volume(volume)
_delete_volume(volume)
chunk_size = self.configuration.rbd_store_chunk_size * units.Mi
order = int(math.log(chunk_size, 2))