Snapshot: offload glance upload in a native thread

Execute glance upload in a native thread as it may block the current
coroutine until it completes.

Despite the fact we use eventlet monkey_patching [1] to achieve cooperative
yielding for network IO, file IO on busy file system may still get
nova-compute hanging.

Stick those IO in a native thread using eventlet tpool.execute() [2]
avoid this issue.

[1] https://eventlet.net/doc/patching.html
[2] https://eventlet.net/doc/threading.html

Closes-Bug: #1874032
Change-Id: I8dbc579e0037969aab4f2bb500fccfbde4190726

nova/image/glance.py

@@ -492,7 +492,12 @@ class GlanceImageServiceV2(object):
             _reraise_translated_exception()
 
     def _upload_data(self, context, image_id, data):
-        self._client.call(context, 2, 'upload', args=(image_id, data))
+        # NOTE(aarents) offload upload in a native thread as it can block
+        # coroutine in busy environment.
+        utils.tpool_execute(self._client.call,
+                      context, 2, 'upload',
+                      args=(image_id, data))
+
         return self._client.call(context, 2, 'get', args=(image_id,))
 
     def _get_image_create_disk_format_default(self, context):

nova/tests/unit/image/test_glance.py

@@ -1724,11 +1724,13 @@ class TestCreate(test.NoDBTestCase):
 class TestUpdate(test.NoDBTestCase):
 
     """Tests the update method of the GlanceImageServiceV2."""
+    @mock.patch('nova.utils.tpool_execute',
+                side_effect=nova.utils.tpool_execute)
     @mock.patch('nova.image.glance.GlanceImageServiceV2.show')
     @mock.patch('nova.image.glance._translate_from_glance')
     @mock.patch('nova.image.glance._translate_to_glance')
     def test_update_success_v2(
-            self, trans_to_mock, trans_from_mock, show_mock):
+            self, trans_to_mock, trans_from_mock, show_mock, texec_mock):
         image = {
             'id': mock.sentinel.image_id,
             'name': mock.sentinel.name,
@@ -1777,6 +1779,10 @@ class TestUpdate(test.NoDBTestCase):
                        data=mock.sentinel.data)
 
         self.assertEqual(3, client.call.call_count)
+        texec_mock.assert_called_once_with(
+            client.call, ctx, 2, 'upload',
+            args=(mock.sentinel.image_id,
+                  mock.sentinel.data))
 
     @mock.patch('nova.image.glance.GlanceImageServiceV2.show')
     @mock.patch('nova.image.glance._translate_from_glance')

nova/tests/unit/test_utils.py

@@ -225,6 +225,12 @@ class GenericUtilsTestCase(test.NoDBTestCase):
         utils.ssh_execute('remotehost', 'ls', '-l')
         mock_execute.assert_called_once_with(*expected_args)
 
+    @mock.patch('nova.utils.generate_uid')
+    def test_tpool_execute(self, mock_generate):
+        expected_kargs = {'size': 12}
+        utils.tpool_execute(utils.generate_uid, 'mytopic', size=12)
+        mock_generate.assert_called_once_with('mytopic', **expected_kargs)
+
     def test_generate_hostid(self):
         host = 'host'
         project_id = '9b9e3c847e904b0686e8ffb20e4c6381'

nova/utils.py

@@ -698,6 +698,11 @@ def spawn_n(func, *args, **kwargs):
     eventlet.spawn_n(context_wrapper, *args, **kwargs)
 
 
+def tpool_execute(func, *args, **kwargs):
+    """Run func in a native thread"""
+    eventlet.tpool.execute(func, *args, **kwargs)
+
+
 def is_none_string(val):
     """Check if a string represents a None value.
     """

releasenotes/notes/bug-1874032-2b01ed05bc7f6f8d.yaml

@@ -0,0 +1,8 @@
+fixes:
+  - |
+    This release contains a fix for `bug 1874032`_ which delegates snapshot
+    upload into a dedicated thread. This ensures nova compute service stability
+    on busy environment during snapshot, when concurrent snapshots or any
+    other tasks slow down storage performance.
+
+    .. _bug 1874032: https://launchpad.net/bugs/1874032