Snapshot: offload glance upload in a native thread

Execute glance upload in a native thread as it may block the current
coroutine until it completes.

Despite the fact we use eventlet monkey_patching [1] to achieve cooperative
yielding for network IO, file IO on busy file system may still get
nova-compute hanging.

Stick those IO in a native thread using eventlet tpool.execute() [2]
avoid this issue.

[1] https://eventlet.net/doc/patching.html
[2] https://eventlet.net/doc/threading.html

Closes-Bug: #1874032
Change-Id: I8dbc579e0037969aab4f2bb500fccfbde4190726
This commit is contained in:
Alexandre Arents 2020-06-09 14:17:37 +00:00
parent 6bb0c4fdab
commit be9b735847
5 changed files with 32 additions and 2 deletions

View File

@ -492,7 +492,12 @@ class GlanceImageServiceV2(object):
_reraise_translated_exception()
def _upload_data(self, context, image_id, data):
self._client.call(context, 2, 'upload', args=(image_id, data))
# NOTE(aarents) offload upload in a native thread as it can block
# coroutine in busy environment.
utils.tpool_execute(self._client.call,
context, 2, 'upload',
args=(image_id, data))
return self._client.call(context, 2, 'get', args=(image_id,))
def _get_image_create_disk_format_default(self, context):

View File

@ -1724,11 +1724,13 @@ class TestCreate(test.NoDBTestCase):
class TestUpdate(test.NoDBTestCase):
"""Tests the update method of the GlanceImageServiceV2."""
@mock.patch('nova.utils.tpool_execute',
side_effect=nova.utils.tpool_execute)
@mock.patch('nova.image.glance.GlanceImageServiceV2.show')
@mock.patch('nova.image.glance._translate_from_glance')
@mock.patch('nova.image.glance._translate_to_glance')
def test_update_success_v2(
self, trans_to_mock, trans_from_mock, show_mock):
self, trans_to_mock, trans_from_mock, show_mock, texec_mock):
image = {
'id': mock.sentinel.image_id,
'name': mock.sentinel.name,
@ -1777,6 +1779,10 @@ class TestUpdate(test.NoDBTestCase):
data=mock.sentinel.data)
self.assertEqual(3, client.call.call_count)
texec_mock.assert_called_once_with(
client.call, ctx, 2, 'upload',
args=(mock.sentinel.image_id,
mock.sentinel.data))
@mock.patch('nova.image.glance.GlanceImageServiceV2.show')
@mock.patch('nova.image.glance._translate_from_glance')

View File

@ -225,6 +225,12 @@ class GenericUtilsTestCase(test.NoDBTestCase):
utils.ssh_execute('remotehost', 'ls', '-l')
mock_execute.assert_called_once_with(*expected_args)
@mock.patch('nova.utils.generate_uid')
def test_tpool_execute(self, mock_generate):
expected_kargs = {'size': 12}
utils.tpool_execute(utils.generate_uid, 'mytopic', size=12)
mock_generate.assert_called_once_with('mytopic', **expected_kargs)
def test_generate_hostid(self):
host = 'host'
project_id = '9b9e3c847e904b0686e8ffb20e4c6381'

View File

@ -698,6 +698,11 @@ def spawn_n(func, *args, **kwargs):
eventlet.spawn_n(context_wrapper, *args, **kwargs)
def tpool_execute(func, *args, **kwargs):
"""Run func in a native thread"""
eventlet.tpool.execute(func, *args, **kwargs)
def is_none_string(val):
"""Check if a string represents a None value.
"""

View File

@ -0,0 +1,8 @@
fixes:
- |
This release contains a fix for `bug 1874032`_ which delegates snapshot
upload into a dedicated thread. This ensures nova compute service stability
on busy environment during snapshot, when concurrent snapshots or any
other tasks slow down storage performance.
.. _bug 1874032: https://launchpad.net/bugs/1874032