openstack/cinder
Code Issues Proposed changes

Merge "Dell SC: Retry unhandled exception REST Gets"

Browse Source
This commit is contained in:
Jenkins
2016-06-13 16:23:01 +00:00
committed by Gerrit Code Review
parent 0f72ff555d 2c54480476
commit 1c9b6f116f
3 changed files with 269 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
cinder/exception.py
View File

@@ -792,6 +792,11 @@ class VolumeDeviceNotFound(CinderException):
# Driver specific exceptions
# Dell
class DellDriverRetryableException(VolumeBackendAPIException):
message = _("Retryable Dell Exception encountered")
# Pure Storage
class PureDriverException(VolumeDriverException):
message = _("Pure Storage Cinder driver failure: %(reason)s")

188
cinder/tests/unit/test_dellscapi.py
View File

@@ -14,6 +14,7 @@
import ddt
import mock
import requests
from requests import models
import uuid
@@ -6853,3 +6854,190 @@ class DellSCSanAPIConnectionTestCase(test.TestCase):
mock_post):
self.scapi.close_connection()
self.assertTrue(mock_post.called)
class DellHttpClientTestCase(test.TestCase):
"""DellSCSanAPIConnectionTestCase
Class to test the Storage Center API connection using Mock.
"""
ASYNCTASK = {"state": "Running",
"methodName": "GetScUserPreferencesDefaults",
"error": "",
"started": True,
"userName": "",
"localizedError": "",
"returnValue": "https://localhost:3033/api/rest/"
"ApiConnection/AsyncTask/1418394170395",
"storageCenter": 0,
"errorState": "None",
"successful": False,
"stepMessage": "Running Method [Object: ScUserPreferences] "
"[Method: GetScUserPreferencesDefaults]",
"localizedStepMessage": "",
"warningList": [],
"totalSteps": 2,
"timeFinished": "1969-12-31T18:00:00-06:00",
"timeStarted": "2015-01-07T14:07:10-06:00",
"currentStep": 1,
"objectTypeName": "ScUserPreferences",
"objectType": "AsyncTask",
"instanceName": "1418394170395",
"instanceId": "1418394170395"}
# Create a Response object that indicates OK
response_ok = models.Response()
response_ok.status_code = 200
response_ok.reason = u'ok'
RESPONSE_200 = response_ok
# Create a Response object with no content
response_nc = models.Response()
response_nc.status_code = 204
response_nc.reason = u'duplicate'
RESPONSE_204 = response_nc
# Create a Response object is a pure error.
response_bad = models.Response()
response_bad.status_code = 400
response_bad.reason = u'bad request'
RESPONSE_400 = response_bad
def setUp(self):
super(DellHttpClientTestCase, self).setUp()
self.host = 'localhost'
self.port = '3033'
self.user = 'johnnyuser'
self.password = 'password'
self.verify = False
self.apiversion = '3.1'
self.httpclient = dell_storagecenter_api.HttpClient(
self.host, self.port, self.user, self.password,
self.verify, self.apiversion)
def test_get_async_url(self):
url = self.httpclient._get_async_url(self.ASYNCTASK)
self.assertEqual('api/rest/ApiConnection/AsyncTask/1418394170395', url)
def test_get_async_url_no_id_on_url(self):
badTask = self.ASYNCTASK.copy()
badTask['returnValue'] = ('https://localhost:3033/api/rest/'
'ApiConnection/AsyncTask/')
url = self.httpclient._get_async_url(badTask)
self.assertEqual('api/rest/ApiConnection/AsyncTask/1418394170395', url)
def test_get_async_url_none(self):
self.assertRaises(AttributeError, self.httpclient._get_async_url, None)
def test_get_async_url_no_id(self):
badTask = self.ASYNCTASK.copy()
badTask['returnValue'] = ('https://localhost:3033/api/rest/'
'ApiConnection/AsyncTask/')
badTask['instanceId'] = ''
self.assertRaises(exception.VolumeBackendAPIException,
self.httpclient._get_async_url, badTask)
def test_rest_ret(self):
rest_response = self.RESPONSE_200
response = self.httpclient._rest_ret(rest_response, False)
self.assertEqual(self.RESPONSE_200, response)
@mock.patch.object(dell_storagecenter_api.HttpClient,
'_wait_for_async_complete',
return_value=RESPONSE_200)
def test_rest_ret_async(self,
mock_wait_for_async_complete):
mock_rest_response = mock.MagicMock()
mock_rest_response.status_code = 202
response = self.httpclient._rest_ret(mock_rest_response, True)
self.assertEqual(self.RESPONSE_200, response)
self.assertTrue(mock_wait_for_async_complete.called)
def test_rest_ret_async_error(self):
mock_rest_response = mock.MagicMock()
mock_rest_response.status_code = 400
self.assertRaises(exception.VolumeBackendAPIException,
self.httpclient._rest_ret, mock_rest_response, True)
@mock.patch.object(dell_storagecenter_api.HttpClient,
'get',
return_value=RESPONSE_200)
def test_wait_for_async_complete(self,
mock_get):
ret = self.httpclient._wait_for_async_complete(self.ASYNCTASK)
self.assertEqual(self.RESPONSE_200, ret)
@mock.patch.object(dell_storagecenter_api.HttpClient,
'_get_async_url',
return_value=None)
def test_wait_for_async_complete_bad_url(self,
mock_get_async_url):
ret = self.httpclient._wait_for_async_complete(self.ASYNCTASK)
self.assertIsNone(ret)
@mock.patch.object(dell_storagecenter_api.HttpClient,
'get',
return_value=RESPONSE_400)
def test_wait_for_async_complete_bad_result(self,
mock_get):
ret = self.httpclient._wait_for_async_complete(self.ASYNCTASK)
self.assertEqual(self.RESPONSE_400, ret)
@mock.patch.object(dell_storagecenter_api.HttpClient,
'get',
return_value=RESPONSE_200)
def test_wait_for_async_complete_loop(self,
mock_get):
mock_response = mock.MagicMock()
mock_response.content = mock.MagicMock()
mock_response.json = mock.MagicMock()
mock_response.json.side_effect = [self.ASYNCTASK,
{'objectType': 'ScVol'}]
ret = self.httpclient._wait_for_async_complete(self.ASYNCTASK)
self.assertEqual(self.RESPONSE_200, ret)
@mock.patch.object(dell_storagecenter_api.HttpClient,
'get')
def test_wait_for_async_complete_get_raises(self,
mock_get):
mock_get.side_effect = (exception.DellDriverRetryableException())
self.assertRaises(exception.VolumeBackendAPIException,
self.httpclient._wait_for_async_complete,
self.ASYNCTASK)
@mock.patch.object(dell_storagecenter_api.HttpClient,
'_rest_ret',
return_value=RESPONSE_200)
@mock.patch.object(requests.Session,
'get',
return_value=RESPONSE_200)
def test_get(self,
mock_get,
mock_rest_ret):
ret = self.httpclient.get('url', False)
self.assertEqual(self.RESPONSE_200, ret)
mock_rest_ret.assert_called_once_with(self.RESPONSE_200, False)
expected_headers = self.httpclient.header.copy()
mock_get.assert_called_once_with('https://localhost:3033/api/rest/url',
headers=expected_headers,
verify=False)
@mock.patch.object(dell_storagecenter_api.HttpClient,
'_rest_ret',
return_value=RESPONSE_200)
@mock.patch.object(requests.Session,
'get',
return_value=RESPONSE_200)
def test_get_async(self,
mock_get,
mock_rest_ret):
ret = self.httpclient.get('url', True)
self.assertEqual(self.RESPONSE_200, ret)
mock_rest_ret.assert_called_once_with(self.RESPONSE_200, True)
expected_headers = self.httpclient.header.copy()
expected_headers['async'] = True
mock_get.assert_called_once_with('https://localhost:3033/api/rest/url',
headers=expected_headers,
verify=False)

111
cinder/volume/drivers/dell/dell_storagecenter_api.py
View File

@@ -87,7 +87,7 @@ class HttpClient(object):
should be turned on or not.
:param apiversion: Dell API version.
"""
self.baseUrl = 'https://%s:%s/api/rest/' % (host, port)
self.baseUrl = 'https://%s:%s/' % (host, port)
self.session = requests.Session()
self.session.auth = (user, password)
@@ -110,7 +110,11 @@ class HttpClient(object):
self.session.close()
def __formatUrl(self, url):
return '%s%s' % (self.baseUrl, url if url[0] != '/' else url[1:])
baseurl = self.baseUrl
# Some url sources have api/rest and some don't. Handle.
if 'api/rest' not in url:
baseurl += 'api/rest/'
return '%s%s' % (baseurl, url if url[0] != '/' else url[1:])
def _get_header(self, async):
if async:
@@ -119,25 +123,53 @@ class HttpClient(object):
return header
return self.header
def _get_async_url(self, asyncTask):
"""Handle a bug in SC API that gives a full url."""
try:
# strip off the https.
url = asyncTask.get('returnValue').split(
'https://')[1].split('/', 1)[1]
except IndexError:
url = asyncTask.get('returnValue')
# Check for incomplete url error case.
if url.endswith('/'):
# Try to fix.
id = asyncTask.get('instanceId')
if id:
# We have an id so note the error and add the id.
LOG.debug('_get_async_url: url format error. (%s)', asyncTask)
url = url + id
else:
# No hope.
LOG.error(_LE('_get_async_url: Bogus return url %s'), url)
raise exception.VolumeBackendAPIException(
message=_('_get_async_url: Invalid URL.'))
return url
def _wait_for_async_complete(self, asyncTask):
url = asyncTask.get('returnValue')
url = self._get_async_url(asyncTask)
while True and url:
try:
r = self.session.get(url, headers=self.header,
verify=self.verify)
r = self.get(url)
# We can leave this loop for a variety of reasons.
# Nothing returned.
# r.content blanks.
# Object returned switches to one without objectType or with
# a different objectType.
if r and r.content:
content = r.json()
if content.get('objectType') == 'AsyncTask':
url = content.get('returnValue')
eventlet.sleep(1)
continue
if not StorageCenterApi._check_result(r):
LOG.debug('Async error: status_code: %s', r.status_code)
else:
# In theory we have a good run.
if r.content:
content = r.json()
if content.get('objectType') == 'AsyncTask':
url = self._get_async_url(content)
eventlet.sleep(1)
continue
else:
LOG.debug('Async debug: r.content is None')
return r
except exception:
except Exception:
methodname = asyncTask.get('methodName')
objectTypeName = asyncTask.get('objectTypeName')
msg = (_('Async error: Unable to retreive %(obj)s '
@@ -165,12 +197,17 @@ class HttpClient(object):
raise exception.VolumeBackendAPIException(message=msg)
return rest_response
@utils.retry(exceptions=(requests.ConnectionError,))
@utils.retry(exceptions=(requests.ConnectionError,
exception.DellDriverRetryableException))
def get(self, url, async=False):
LOG.debug('get: %(url)s', {'url': url})
return self._rest_ret(self.session.get(self.__formatUrl(url),
headers=self._get_header(async),
verify=self.verify), async)
rest_response = self._rest_ret(self.session.get(
self.__formatUrl(url), headers=self._get_header(async),
verify=self.verify), async)
if rest_response and rest_response.status_code == 400 and (
'Unhandled Exception' in rest_response.text):
raise exception.DellDriverRetryableException()
return rest_response
@utils.retry(exceptions=(requests.ConnectionError,))
def post(self, url, payload, async=False):
@@ -344,26 +381,30 @@ class StorageCenterApi(object):
:param rest_response: The result from a REST API call.
:returns: ``True`` if success, ``False`` otherwise.
"""
if 200 <= rest_response.status_code < 300:
# API call was a normal success
return True
if rest_response:
if 200 <= rest_response.status_code < 300:
# API call was a normal success
return True
# Some versions return this as a dict.
try:
response_text = rest_response.text['result']
except Exception:
# We do not care why that failed. Just use the text.
response_text = rest_response.text
# Some versions return this as a dict.
try:
response_json = rest_response.json()
response_text = response_json.text['result']
except Exception:
# We do not care why that failed. Just use the text.
response_text = rest_response.text
LOG.debug('REST call result:\n'
'\tUrl: %(url)s\n'
'\tCode: %(code)d\n'
'\tReason: %(reason)s\n'
'\tText: %(text)s',
{'url': rest_response.url,
'code': rest_response.status_code,
'reason': rest_response.reason,
'text': response_text})
LOG.debug('REST call result:\n'
'\tUrl: %(url)s\n'
'\tCode: %(code)d\n'
'\tReason: %(reason)s\n'
'\tText: %(text)s',
{'url': rest_response.url,
'code': rest_response.status_code,
'reason': rest_response.reason,
'text': response_text})
else:
LOG.warning(_LW('Failed to get REST call result.'))
return False
@staticmethod
@@ -2627,7 +2668,7 @@ class StorageCenterApi(object):
pf.append('scSerialNumber', ssn)
pf.append('name', foldername)
r = self.client.post('StorageCenter/ScDiskFolder/GetList',
pf.payload, True)
pf.payload)
if self._check_result(r):
try:
# Go for broke.