RBD: Wrap RBD calls in native threads
librbd methods call lower level C code which runs in native code
and isn't aware about the eventlet threads hence hangs the eventlet
loop until the native code is finished.
This could cause problems when we are creating multiple images
with large size where one call to librados can cause the process to
hang and other operations can starve for execution and error out.
This patch wraps each RBD call in it's own native thread that won't
affect other RBD call from executing.
Note that since glance can run under eventlet *or* native threads
with something like uwsgi, this only delegates to an eventlet worker
when we are running in that environment.
Note: This was originally merged as commit
27ab8a6aeb
, but without the handling
for non-eventlet cases.
Change-Id: I87dd4a129113e6309bee1b35a7a45b2a04353b04
Co-Authored-By: Dan Smith <dms@danplanet.com>
This commit is contained in:
parent
cc39326230
commit
127410cd81
|
@ -22,8 +22,10 @@ import logging
|
|||
import math
|
||||
import urllib
|
||||
|
||||
from eventlet import tpool
|
||||
from oslo_config import cfg
|
||||
from oslo_utils import encodeutils
|
||||
from oslo_utils import eventletutils
|
||||
from oslo_utils import units
|
||||
|
||||
from glance_store import capabilities
|
||||
|
@ -290,6 +292,12 @@ class Store(driver.Store):
|
|||
def get_schemes(self):
|
||||
return ('rbd',)
|
||||
|
||||
def RBDProxy(self):
|
||||
if eventletutils.is_monkey_patched('thread'):
|
||||
return tpool.Proxy(rbd.RBD())
|
||||
else:
|
||||
return rbd.RBD()
|
||||
|
||||
@contextlib.contextmanager
|
||||
def get_connection(self, conffile, rados_id):
|
||||
client = rados.Rados(conffile=conffile, rados_id=rados_id)
|
||||
|
@ -431,12 +439,12 @@ class Store(driver.Store):
|
|||
|
||||
:returns: `glance_store.rbd.StoreLocation` object
|
||||
"""
|
||||
librbd = rbd.RBD()
|
||||
features = conn.conf_get('rbd_default_features')
|
||||
if ((features is None) or (int(features) == 0)):
|
||||
features = rbd.RBD_FEATURE_LAYERING
|
||||
librbd.create(ioctx, image_name, size, order, old_format=False,
|
||||
features=int(features))
|
||||
self.RBDProxy().create(ioctx, image_name, size, order,
|
||||
old_format=False,
|
||||
features=int(features))
|
||||
return StoreLocation({
|
||||
'fsid': fsid,
|
||||
'pool': self.pool,
|
||||
|
@ -496,7 +504,7 @@ class Store(driver.Store):
|
|||
raise exceptions.InUseByStore()
|
||||
|
||||
# Then delete image.
|
||||
rbd.RBD().remove(ioctx, image_name)
|
||||
self.RBDProxy().remove(ioctx, image_name)
|
||||
except rbd.ImageHasSnapshots:
|
||||
log_msg = (_LW("Remove image %(img_name)s failed. "
|
||||
"It has snapshot(s) left.") %
|
||||
|
|
|
@ -692,3 +692,67 @@ class TestStore(base.StoreBaseTest,
|
|||
self.assertEqual(self.called_commands_expected,
|
||||
self.called_commands_actual)
|
||||
super(TestStore, self).tearDown()
|
||||
|
||||
@mock.patch('oslo_utils.eventletutils.is_monkey_patched')
|
||||
def test_create_image_in_native_thread(self, mock_patched):
|
||||
mock_patched.return_value = True
|
||||
# Tests that we use non-0 features from ceph.conf and cast to int.
|
||||
fsid = 'fake'
|
||||
features = '3'
|
||||
conf_get_mock = mock.Mock(return_value=features)
|
||||
conn = mock.Mock(conf_get=conf_get_mock)
|
||||
ioctxt = mock.sentinel.ioctxt
|
||||
name = '1'
|
||||
size = 1024
|
||||
order = 3
|
||||
fake_proxy = mock.MagicMock()
|
||||
fake_rbd = mock.MagicMock()
|
||||
|
||||
with mock.patch.object(rbd_store.tpool, 'Proxy') as tpool_mock, \
|
||||
mock.patch.object(rbd_store.rbd, 'RBD') as rbd_mock:
|
||||
tpool_mock.return_value = fake_proxy
|
||||
rbd_mock.return_value = fake_rbd
|
||||
location = self.store._create_image(
|
||||
fsid, conn, ioctxt, name, size, order)
|
||||
self.assertEqual(fsid, location.specs['fsid'])
|
||||
self.assertEqual(rbd_store.DEFAULT_POOL, location.specs['pool'])
|
||||
self.assertEqual(name, location.specs['image'])
|
||||
self.assertEqual(rbd_store.DEFAULT_SNAPNAME,
|
||||
location.specs['snapshot'])
|
||||
|
||||
tpool_mock.assert_called_once_with(fake_rbd)
|
||||
fake_proxy.create.assert_called_once_with(ioctxt, name, size, order,
|
||||
old_format=False, features=3)
|
||||
|
||||
@mock.patch('oslo_utils.eventletutils.is_monkey_patched')
|
||||
def test_delete_image_in_native_thread(self, mock_patched):
|
||||
mock_patched.return_value = True
|
||||
fake_proxy = mock.MagicMock()
|
||||
fake_rbd = mock.MagicMock()
|
||||
fake_ioctx = mock.MagicMock()
|
||||
|
||||
with mock.patch.object(rbd_store.tpool, 'Proxy') as tpool_mock, \
|
||||
mock.patch.object(rbd_store.rbd, 'RBD') as rbd_mock, \
|
||||
mock.patch.object(self.store, 'get_connection') as mock_conn:
|
||||
|
||||
mock_get_conn = mock_conn.return_value.__enter__.return_value
|
||||
mock_ioctx = mock_get_conn.open_ioctx.return_value.__enter__
|
||||
mock_ioctx.return_value = fake_ioctx
|
||||
tpool_mock.return_value = fake_proxy
|
||||
rbd_mock.return_value = fake_rbd
|
||||
|
||||
self.store._delete_image('fake_pool', self.location.image)
|
||||
|
||||
tpool_mock.assert_called_once_with(fake_rbd)
|
||||
fake_proxy.remove.assert_called_once_with(fake_ioctx,
|
||||
self.location.image)
|
||||
|
||||
@mock.patch.object(rbd_store, 'rbd')
|
||||
@mock.patch.object(rbd_store, 'tpool')
|
||||
@mock.patch('oslo_utils.eventletutils.is_monkey_patched')
|
||||
def test_rbd_proxy(self, mock_patched, mock_tpool, mock_rbd):
|
||||
mock_patched.return_value = False
|
||||
self.assertEqual(mock_rbd.RBD(), self.store.RBDProxy())
|
||||
|
||||
mock_patched.return_value = True
|
||||
self.assertEqual(mock_tpool.Proxy.return_value, self.store.RBDProxy())
|
||||
|
|
Loading…
Reference in New Issue