Merge "Add retry to cinder API calls related to volume detach" into stable/queens
This commit is contained in:
commit
77c13e0b76
|
@ -14,6 +14,7 @@
|
||||||
# under the License.
|
# under the License.
|
||||||
|
|
||||||
from cinderclient import api_versions as cinder_api_versions
|
from cinderclient import api_versions as cinder_api_versions
|
||||||
|
from cinderclient import apiclient as cinder_apiclient
|
||||||
from cinderclient import exceptions as cinder_exception
|
from cinderclient import exceptions as cinder_exception
|
||||||
from cinderclient.v2 import limits as cinder_limits
|
from cinderclient.v2 import limits as cinder_limits
|
||||||
from keystoneauth1 import loading as ks_loading
|
from keystoneauth1 import loading as ks_loading
|
||||||
|
@ -540,6 +541,38 @@ class CinderApiTestCase(test.NoDBTestCase):
|
||||||
mock_cinderclient.assert_called_once_with(self.ctx, '3.44',
|
mock_cinderclient.assert_called_once_with(self.ctx, '3.44',
|
||||||
skip_version_check=True)
|
skip_version_check=True)
|
||||||
|
|
||||||
|
@mock.patch('nova.volume.cinder.cinderclient',
|
||||||
|
side_effect=cinder_apiclient.exceptions.InternalServerError)
|
||||||
|
def test_attachment_delete_internal_server_error(self, mock_cinderclient):
|
||||||
|
|
||||||
|
self.assertRaises(cinder_apiclient.exceptions.InternalServerError,
|
||||||
|
self.api.attachment_delete,
|
||||||
|
self.ctx, uuids.attachment_id)
|
||||||
|
|
||||||
|
self.assertEqual(5, mock_cinderclient.call_count)
|
||||||
|
|
||||||
|
@mock.patch('nova.volume.cinder.cinderclient')
|
||||||
|
def test_attachment_delete_internal_server_error_do_not_raise(
|
||||||
|
self, mock_cinderclient):
|
||||||
|
# generate exception, and then have a normal return on the next retry
|
||||||
|
mock_cinderclient.return_value.attachments.delete.side_effect = [
|
||||||
|
cinder_apiclient.exceptions.InternalServerError, None]
|
||||||
|
|
||||||
|
attachment_id = uuids.attachment
|
||||||
|
self.api.attachment_delete(self.ctx, attachment_id)
|
||||||
|
|
||||||
|
self.assertEqual(2, mock_cinderclient.call_count)
|
||||||
|
|
||||||
|
@mock.patch('nova.volume.cinder.cinderclient',
|
||||||
|
side_effect=cinder_exception.BadRequest(code=400))
|
||||||
|
def test_attachment_delete_bad_request_exception(self, mock_cinderclient):
|
||||||
|
|
||||||
|
self.assertRaises(exception.InvalidInput,
|
||||||
|
self.api.attachment_delete,
|
||||||
|
self.ctx, uuids.attachment_id)
|
||||||
|
|
||||||
|
self.assertEqual(1, mock_cinderclient.call_count)
|
||||||
|
|
||||||
@mock.patch('nova.volume.cinder.cinderclient')
|
@mock.patch('nova.volume.cinder.cinderclient')
|
||||||
def test_attachment_complete(self, mock_cinderclient):
|
def test_attachment_complete(self, mock_cinderclient):
|
||||||
mock_attachments = mock.MagicMock()
|
mock_attachments = mock.MagicMock()
|
||||||
|
@ -629,6 +662,38 @@ class CinderApiTestCase(test.NoDBTestCase):
|
||||||
mock_cinderclient.assert_called_with(self.ctx, microversion=None)
|
mock_cinderclient.assert_called_with(self.ctx, microversion=None)
|
||||||
mock_volumes.detach.assert_called_once_with('id1', 'fakeid')
|
mock_volumes.detach.assert_called_once_with('id1', 'fakeid')
|
||||||
|
|
||||||
|
@mock.patch('nova.volume.cinder.cinderclient',
|
||||||
|
side_effect=cinder_apiclient.exceptions.InternalServerError)
|
||||||
|
def test_detach_internal_server_error(self, mock_cinderclient):
|
||||||
|
|
||||||
|
self.assertRaises(cinder_apiclient.exceptions.InternalServerError,
|
||||||
|
self.api.detach,
|
||||||
|
self.ctx, 'id1', instance_uuid='fake_uuid')
|
||||||
|
|
||||||
|
self.assertEqual(5, mock_cinderclient.call_count)
|
||||||
|
|
||||||
|
@mock.patch('nova.volume.cinder.cinderclient')
|
||||||
|
def test_detach_internal_server_error_do_not_raise(
|
||||||
|
self, mock_cinderclient):
|
||||||
|
# generate exception, and then have a normal return on the next retry
|
||||||
|
mock_cinderclient.return_value.volumes.detach.side_effect = [
|
||||||
|
cinder_apiclient.exceptions.InternalServerError, None]
|
||||||
|
|
||||||
|
self.api.detach(self.ctx, 'id1', instance_uuid='fake_uuid',
|
||||||
|
attachment_id='fakeid')
|
||||||
|
|
||||||
|
self.assertEqual(2, mock_cinderclient.call_count)
|
||||||
|
|
||||||
|
@mock.patch('nova.volume.cinder.cinderclient',
|
||||||
|
side_effect=cinder_exception.BadRequest(code=400))
|
||||||
|
def test_detach_bad_request_exception(self, mock_cinderclient):
|
||||||
|
|
||||||
|
self.assertRaises(exception.InvalidInput,
|
||||||
|
self.api.detach,
|
||||||
|
self.ctx, 'id1', instance_uuid='fake_uuid')
|
||||||
|
|
||||||
|
self.assertEqual(1, mock_cinderclient.call_count)
|
||||||
|
|
||||||
@mock.patch('nova.volume.cinder.cinderclient')
|
@mock.patch('nova.volume.cinder.cinderclient')
|
||||||
def test_attachment_get(self, mock_cinderclient):
|
def test_attachment_get(self, mock_cinderclient):
|
||||||
mock_attachment = mock.MagicMock()
|
mock_attachment = mock.MagicMock()
|
||||||
|
@ -748,6 +813,38 @@ class CinderApiTestCase(test.NoDBTestCase):
|
||||||
mock_volumes.terminate_connection.assert_called_once_with('id1',
|
mock_volumes.terminate_connection.assert_called_once_with('id1',
|
||||||
'connector')
|
'connector')
|
||||||
|
|
||||||
|
@mock.patch('nova.volume.cinder.cinderclient',
|
||||||
|
side_effect=cinder_apiclient.exceptions.InternalServerError)
|
||||||
|
def test_terminate_connection_internal_server_error(
|
||||||
|
self, mock_cinderclient):
|
||||||
|
self.assertRaises(cinder_apiclient.exceptions.InternalServerError,
|
||||||
|
self.api.terminate_connection,
|
||||||
|
self.ctx, 'id1', 'connector')
|
||||||
|
|
||||||
|
self.assertEqual(5, mock_cinderclient.call_count)
|
||||||
|
|
||||||
|
@mock.patch('nova.volume.cinder.cinderclient')
|
||||||
|
def test_terminate_connection_internal_server_error_do_not_raise(
|
||||||
|
self, mock_cinderclient):
|
||||||
|
# generate exception, and then have a normal return on the next retry
|
||||||
|
mock_cinderclient.return_value.volumes.terminate_connection.\
|
||||||
|
side_effect = [cinder_apiclient.exceptions.InternalServerError,
|
||||||
|
None]
|
||||||
|
|
||||||
|
self.api.terminate_connection(self.ctx, 'id1', 'connector')
|
||||||
|
|
||||||
|
self.assertEqual(2, mock_cinderclient.call_count)
|
||||||
|
|
||||||
|
@mock.patch('nova.volume.cinder.cinderclient',
|
||||||
|
side_effect=cinder_exception.BadRequest(code=400))
|
||||||
|
def test_terminate_connection_bad_request_exception(
|
||||||
|
self, mock_cinderclient):
|
||||||
|
self.assertRaises(exception.InvalidInput,
|
||||||
|
self.api.terminate_connection,
|
||||||
|
self.ctx, 'id1', 'connector')
|
||||||
|
|
||||||
|
self.assertEqual(1, mock_cinderclient.call_count)
|
||||||
|
|
||||||
@mock.patch('nova.volume.cinder.cinderclient')
|
@mock.patch('nova.volume.cinder.cinderclient')
|
||||||
def test_delete(self, mock_cinderclient):
|
def test_delete(self, mock_cinderclient):
|
||||||
mock_volumes = mock.MagicMock()
|
mock_volumes = mock.MagicMock()
|
||||||
|
|
|
@ -24,6 +24,7 @@ import functools
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
from cinderclient import api_versions as cinder_api_versions
|
from cinderclient import api_versions as cinder_api_versions
|
||||||
|
from cinderclient import apiclient as cinder_apiclient
|
||||||
from cinderclient import client as cinder_client
|
from cinderclient import client as cinder_client
|
||||||
from cinderclient import exceptions as cinder_exception
|
from cinderclient import exceptions as cinder_exception
|
||||||
from keystoneauth1 import exceptions as keystone_exception
|
from keystoneauth1 import exceptions as keystone_exception
|
||||||
|
@ -33,6 +34,7 @@ from oslo_serialization import jsonutils
|
||||||
from oslo_utils import encodeutils
|
from oslo_utils import encodeutils
|
||||||
from oslo_utils import excutils
|
from oslo_utils import excutils
|
||||||
from oslo_utils import strutils
|
from oslo_utils import strutils
|
||||||
|
import retrying
|
||||||
import six
|
import six
|
||||||
from six.moves import urllib
|
from six.moves import urllib
|
||||||
|
|
||||||
|
@ -543,6 +545,9 @@ class API(object):
|
||||||
mountpoint, mode=mode)
|
mountpoint, mode=mode)
|
||||||
|
|
||||||
@translate_volume_exception
|
@translate_volume_exception
|
||||||
|
@retrying.retry(stop_max_attempt_number=5,
|
||||||
|
retry_on_exception=lambda e:
|
||||||
|
type(e) == cinder_apiclient.exceptions.InternalServerError)
|
||||||
def detach(self, context, volume_id, instance_uuid=None,
|
def detach(self, context, volume_id, instance_uuid=None,
|
||||||
attachment_id=None):
|
attachment_id=None):
|
||||||
client = cinderclient(context)
|
client = cinderclient(context)
|
||||||
|
@ -606,6 +611,9 @@ class API(object):
|
||||||
exc.code if hasattr(exc, 'code') else None)})
|
exc.code if hasattr(exc, 'code') else None)})
|
||||||
|
|
||||||
@translate_volume_exception
|
@translate_volume_exception
|
||||||
|
@retrying.retry(stop_max_attempt_number=5,
|
||||||
|
retry_on_exception=lambda e:
|
||||||
|
type(e) == cinder_apiclient.exceptions.InternalServerError)
|
||||||
def terminate_connection(self, context, volume_id, connector):
|
def terminate_connection(self, context, volume_id, connector):
|
||||||
return cinderclient(context).volumes.terminate_connection(volume_id,
|
return cinderclient(context).volumes.terminate_connection(volume_id,
|
||||||
connector)
|
connector)
|
||||||
|
@ -844,6 +852,9 @@ class API(object):
|
||||||
'code': getattr(ex, 'code', None)})
|
'code': getattr(ex, 'code', None)})
|
||||||
|
|
||||||
@translate_attachment_exception
|
@translate_attachment_exception
|
||||||
|
@retrying.retry(stop_max_attempt_number=5,
|
||||||
|
retry_on_exception=lambda e:
|
||||||
|
type(e) == cinder_apiclient.exceptions.InternalServerError)
|
||||||
def attachment_delete(self, context, attachment_id):
|
def attachment_delete(self, context, attachment_id):
|
||||||
try:
|
try:
|
||||||
cinderclient(
|
cinderclient(
|
||||||
|
|
Loading…
Reference in New Issue