RBD: Wrap RBD calls in native threads

librbd methods call lower level C code which runs in native code
and isn't aware about the eventlet threads hence hangs the eventlet
loop until the native code is finished.
This could cause problems when we are creating multiple images
with large size where one call to librados can cause the process to
hang and other operations can starve for execution and error out.
This patch wraps each RBD call in it's own native thread that won't
affect other RBD call from executing.

Note that since glance can run under eventlet *or* native threads
with something like uwsgi, this only delegates to an eventlet worker
when we are running in that environment.

Note: This was originally merged as commit
27ab8a6aeb, but without the handling
for non-eventlet cases.

Change-Id: I87dd4a129113e6309bee1b35a7a45b2a04353b04
Co-Authored-By: Dan Smith <dms@danplanet.com>
This commit is contained in:
Rajat Dhasmana 2022-10-13 14:05:02 +00:00 committed by Dan Smith
parent cc39326230
commit 127410cd81
2 changed files with 76 additions and 4 deletions

View File

@ -22,8 +22,10 @@ import logging
import math
import urllib
from eventlet import tpool
from oslo_config import cfg
from oslo_utils import encodeutils
from oslo_utils import eventletutils
from oslo_utils import units
from glance_store import capabilities
@ -290,6 +292,12 @@ class Store(driver.Store):
def get_schemes(self):
return ('rbd',)
def RBDProxy(self):
if eventletutils.is_monkey_patched('thread'):
return tpool.Proxy(rbd.RBD())
else:
return rbd.RBD()
@contextlib.contextmanager
def get_connection(self, conffile, rados_id):
client = rados.Rados(conffile=conffile, rados_id=rados_id)
@ -431,12 +439,12 @@ class Store(driver.Store):
:returns: `glance_store.rbd.StoreLocation` object
"""
librbd = rbd.RBD()
features = conn.conf_get('rbd_default_features')
if ((features is None) or (int(features) == 0)):
features = rbd.RBD_FEATURE_LAYERING
librbd.create(ioctx, image_name, size, order, old_format=False,
features=int(features))
self.RBDProxy().create(ioctx, image_name, size, order,
old_format=False,
features=int(features))
return StoreLocation({
'fsid': fsid,
'pool': self.pool,
@ -496,7 +504,7 @@ class Store(driver.Store):
raise exceptions.InUseByStore()
# Then delete image.
rbd.RBD().remove(ioctx, image_name)
self.RBDProxy().remove(ioctx, image_name)
except rbd.ImageHasSnapshots:
log_msg = (_LW("Remove image %(img_name)s failed. "
"It has snapshot(s) left.") %

View File

@ -692,3 +692,67 @@ class TestStore(base.StoreBaseTest,
self.assertEqual(self.called_commands_expected,
self.called_commands_actual)
super(TestStore, self).tearDown()
@mock.patch('oslo_utils.eventletutils.is_monkey_patched')
def test_create_image_in_native_thread(self, mock_patched):
mock_patched.return_value = True
# Tests that we use non-0 features from ceph.conf and cast to int.
fsid = 'fake'
features = '3'
conf_get_mock = mock.Mock(return_value=features)
conn = mock.Mock(conf_get=conf_get_mock)
ioctxt = mock.sentinel.ioctxt
name = '1'
size = 1024
order = 3
fake_proxy = mock.MagicMock()
fake_rbd = mock.MagicMock()
with mock.patch.object(rbd_store.tpool, 'Proxy') as tpool_mock, \
mock.patch.object(rbd_store.rbd, 'RBD') as rbd_mock:
tpool_mock.return_value = fake_proxy
rbd_mock.return_value = fake_rbd
location = self.store._create_image(
fsid, conn, ioctxt, name, size, order)
self.assertEqual(fsid, location.specs['fsid'])
self.assertEqual(rbd_store.DEFAULT_POOL, location.specs['pool'])
self.assertEqual(name, location.specs['image'])
self.assertEqual(rbd_store.DEFAULT_SNAPNAME,
location.specs['snapshot'])
tpool_mock.assert_called_once_with(fake_rbd)
fake_proxy.create.assert_called_once_with(ioctxt, name, size, order,
old_format=False, features=3)
@mock.patch('oslo_utils.eventletutils.is_monkey_patched')
def test_delete_image_in_native_thread(self, mock_patched):
mock_patched.return_value = True
fake_proxy = mock.MagicMock()
fake_rbd = mock.MagicMock()
fake_ioctx = mock.MagicMock()
with mock.patch.object(rbd_store.tpool, 'Proxy') as tpool_mock, \
mock.patch.object(rbd_store.rbd, 'RBD') as rbd_mock, \
mock.patch.object(self.store, 'get_connection') as mock_conn:
mock_get_conn = mock_conn.return_value.__enter__.return_value
mock_ioctx = mock_get_conn.open_ioctx.return_value.__enter__
mock_ioctx.return_value = fake_ioctx
tpool_mock.return_value = fake_proxy
rbd_mock.return_value = fake_rbd
self.store._delete_image('fake_pool', self.location.image)
tpool_mock.assert_called_once_with(fake_rbd)
fake_proxy.remove.assert_called_once_with(fake_ioctx,
self.location.image)
@mock.patch.object(rbd_store, 'rbd')
@mock.patch.object(rbd_store, 'tpool')
@mock.patch('oslo_utils.eventletutils.is_monkey_patched')
def test_rbd_proxy(self, mock_patched, mock_tpool, mock_rbd):
mock_patched.return_value = False
self.assertEqual(mock_rbd.RBD(), self.store.RBDProxy())
mock_patched.return_value = True
self.assertEqual(mock_tpool.Proxy.return_value, self.store.RBDProxy())