RBD: Wrap RBD calls in native threads
librbd methods call lower level C code which runs in native code
and isn't aware about the eventlet threads hence hangs the eventlet
loop until the native code is finished.
This could cause problems when we are creating multiple images
with large size where one call to librados can cause the process to
hang and other operations can starve for execution and error out.
This patch wraps each RBD call in it's own native thread that won't
affect other RBD call from executing.
Note that since glance can run under eventlet *or* native threads
with something like uwsgi, this only delegates to an eventlet worker
when we are running in that environment.
Note: This was originally merged as commit
27ab8a6aeb
, but without the handling
for non-eventlet cases.
Change-Id: I87dd4a129113e6309bee1b35a7a45b2a04353b04
Co-Authored-By: Dan Smith <dms@danplanet.com>
This commit is contained in:
parent
cc39326230
commit
127410cd81
|
@ -22,8 +22,10 @@ import logging
|
||||||
import math
|
import math
|
||||||
import urllib
|
import urllib
|
||||||
|
|
||||||
|
from eventlet import tpool
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
from oslo_utils import encodeutils
|
from oslo_utils import encodeutils
|
||||||
|
from oslo_utils import eventletutils
|
||||||
from oslo_utils import units
|
from oslo_utils import units
|
||||||
|
|
||||||
from glance_store import capabilities
|
from glance_store import capabilities
|
||||||
|
@ -290,6 +292,12 @@ class Store(driver.Store):
|
||||||
def get_schemes(self):
|
def get_schemes(self):
|
||||||
return ('rbd',)
|
return ('rbd',)
|
||||||
|
|
||||||
|
def RBDProxy(self):
|
||||||
|
if eventletutils.is_monkey_patched('thread'):
|
||||||
|
return tpool.Proxy(rbd.RBD())
|
||||||
|
else:
|
||||||
|
return rbd.RBD()
|
||||||
|
|
||||||
@contextlib.contextmanager
|
@contextlib.contextmanager
|
||||||
def get_connection(self, conffile, rados_id):
|
def get_connection(self, conffile, rados_id):
|
||||||
client = rados.Rados(conffile=conffile, rados_id=rados_id)
|
client = rados.Rados(conffile=conffile, rados_id=rados_id)
|
||||||
|
@ -431,11 +439,11 @@ class Store(driver.Store):
|
||||||
|
|
||||||
:returns: `glance_store.rbd.StoreLocation` object
|
:returns: `glance_store.rbd.StoreLocation` object
|
||||||
"""
|
"""
|
||||||
librbd = rbd.RBD()
|
|
||||||
features = conn.conf_get('rbd_default_features')
|
features = conn.conf_get('rbd_default_features')
|
||||||
if ((features is None) or (int(features) == 0)):
|
if ((features is None) or (int(features) == 0)):
|
||||||
features = rbd.RBD_FEATURE_LAYERING
|
features = rbd.RBD_FEATURE_LAYERING
|
||||||
librbd.create(ioctx, image_name, size, order, old_format=False,
|
self.RBDProxy().create(ioctx, image_name, size, order,
|
||||||
|
old_format=False,
|
||||||
features=int(features))
|
features=int(features))
|
||||||
return StoreLocation({
|
return StoreLocation({
|
||||||
'fsid': fsid,
|
'fsid': fsid,
|
||||||
|
@ -496,7 +504,7 @@ class Store(driver.Store):
|
||||||
raise exceptions.InUseByStore()
|
raise exceptions.InUseByStore()
|
||||||
|
|
||||||
# Then delete image.
|
# Then delete image.
|
||||||
rbd.RBD().remove(ioctx, image_name)
|
self.RBDProxy().remove(ioctx, image_name)
|
||||||
except rbd.ImageHasSnapshots:
|
except rbd.ImageHasSnapshots:
|
||||||
log_msg = (_LW("Remove image %(img_name)s failed. "
|
log_msg = (_LW("Remove image %(img_name)s failed. "
|
||||||
"It has snapshot(s) left.") %
|
"It has snapshot(s) left.") %
|
||||||
|
|
|
@ -692,3 +692,67 @@ class TestStore(base.StoreBaseTest,
|
||||||
self.assertEqual(self.called_commands_expected,
|
self.assertEqual(self.called_commands_expected,
|
||||||
self.called_commands_actual)
|
self.called_commands_actual)
|
||||||
super(TestStore, self).tearDown()
|
super(TestStore, self).tearDown()
|
||||||
|
|
||||||
|
@mock.patch('oslo_utils.eventletutils.is_monkey_patched')
|
||||||
|
def test_create_image_in_native_thread(self, mock_patched):
|
||||||
|
mock_patched.return_value = True
|
||||||
|
# Tests that we use non-0 features from ceph.conf and cast to int.
|
||||||
|
fsid = 'fake'
|
||||||
|
features = '3'
|
||||||
|
conf_get_mock = mock.Mock(return_value=features)
|
||||||
|
conn = mock.Mock(conf_get=conf_get_mock)
|
||||||
|
ioctxt = mock.sentinel.ioctxt
|
||||||
|
name = '1'
|
||||||
|
size = 1024
|
||||||
|
order = 3
|
||||||
|
fake_proxy = mock.MagicMock()
|
||||||
|
fake_rbd = mock.MagicMock()
|
||||||
|
|
||||||
|
with mock.patch.object(rbd_store.tpool, 'Proxy') as tpool_mock, \
|
||||||
|
mock.patch.object(rbd_store.rbd, 'RBD') as rbd_mock:
|
||||||
|
tpool_mock.return_value = fake_proxy
|
||||||
|
rbd_mock.return_value = fake_rbd
|
||||||
|
location = self.store._create_image(
|
||||||
|
fsid, conn, ioctxt, name, size, order)
|
||||||
|
self.assertEqual(fsid, location.specs['fsid'])
|
||||||
|
self.assertEqual(rbd_store.DEFAULT_POOL, location.specs['pool'])
|
||||||
|
self.assertEqual(name, location.specs['image'])
|
||||||
|
self.assertEqual(rbd_store.DEFAULT_SNAPNAME,
|
||||||
|
location.specs['snapshot'])
|
||||||
|
|
||||||
|
tpool_mock.assert_called_once_with(fake_rbd)
|
||||||
|
fake_proxy.create.assert_called_once_with(ioctxt, name, size, order,
|
||||||
|
old_format=False, features=3)
|
||||||
|
|
||||||
|
@mock.patch('oslo_utils.eventletutils.is_monkey_patched')
|
||||||
|
def test_delete_image_in_native_thread(self, mock_patched):
|
||||||
|
mock_patched.return_value = True
|
||||||
|
fake_proxy = mock.MagicMock()
|
||||||
|
fake_rbd = mock.MagicMock()
|
||||||
|
fake_ioctx = mock.MagicMock()
|
||||||
|
|
||||||
|
with mock.patch.object(rbd_store.tpool, 'Proxy') as tpool_mock, \
|
||||||
|
mock.patch.object(rbd_store.rbd, 'RBD') as rbd_mock, \
|
||||||
|
mock.patch.object(self.store, 'get_connection') as mock_conn:
|
||||||
|
|
||||||
|
mock_get_conn = mock_conn.return_value.__enter__.return_value
|
||||||
|
mock_ioctx = mock_get_conn.open_ioctx.return_value.__enter__
|
||||||
|
mock_ioctx.return_value = fake_ioctx
|
||||||
|
tpool_mock.return_value = fake_proxy
|
||||||
|
rbd_mock.return_value = fake_rbd
|
||||||
|
|
||||||
|
self.store._delete_image('fake_pool', self.location.image)
|
||||||
|
|
||||||
|
tpool_mock.assert_called_once_with(fake_rbd)
|
||||||
|
fake_proxy.remove.assert_called_once_with(fake_ioctx,
|
||||||
|
self.location.image)
|
||||||
|
|
||||||
|
@mock.patch.object(rbd_store, 'rbd')
|
||||||
|
@mock.patch.object(rbd_store, 'tpool')
|
||||||
|
@mock.patch('oslo_utils.eventletutils.is_monkey_patched')
|
||||||
|
def test_rbd_proxy(self, mock_patched, mock_tpool, mock_rbd):
|
||||||
|
mock_patched.return_value = False
|
||||||
|
self.assertEqual(mock_rbd.RBD(), self.store.RBDProxy())
|
||||||
|
|
||||||
|
mock_patched.return_value = True
|
||||||
|
self.assertEqual(mock_tpool.Proxy.return_value, self.store.RBDProxy())
|
||||||
|
|
Loading…
Reference in New Issue