openstack/nova-powervm
Code Issues Proposed changes

Deploy of a VM occasionally fails due to invalid keystone token

Browse Source 
Deploy of virtual machine occasionally fails due to invalid/expired
keystone token with swift and this causes the NVRAM data upload
operation to fail with ClientException error. The results object
returned to nova_powervm will have the error information appended
to it by the swiftclient. Hence, we loop through the results
object to see if there's an authentication failure, and if there
is, we'll retry upload operation, this time with a valid keystone token.

Change-Id: I2779ad6340a58f5c58971eb142ec7b0bd76e2016
Closes-Bug: #1607772
This commit is contained in:
arunmani
2016-08-04 17:13:56 +05:30
parent c3f04d2cda
commit 68d31b0cf6
2 changed files with 49 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
nova_powervm/tests/virt/powervm/nvram/test_swift.py
View File

@@ -16,6 +16,7 @@
import mock
from nova import test
from requests.exceptions import RequestException
from swiftclient import exceptions as swft_exc
from swiftclient import service as swft_srv
@@ -131,18 +132,24 @@ class TestSwiftStore(test.TestCase):
mock.ANY, options=None)
# Test unsuccessful upload
mock_run.return_value[0]['success'] = False
mock_result = [{'success': False,
'error': RequestException('Error Message.')}]
mock_run.return_value = mock_result
self.assertRaises(api.NVRAMUploadException,
self.swift_store._store, powervm.TEST_INST1.uuid,
powervm.TEST_INST1.name, 'data')
# Test retry upload
mock_run.side_effect = [swft_exc.ClientException('Error message.'),
self._build_results(['obj'])]
mock_run.reset_mock()
mock_res_obj = [{'success': False,
'error': swft_exc.
ClientException('Error Message.')}]
mock_run.side_effect = [mock_res_obj, self._build_results(['obj'])]
self.swift_store._store(powervm.TEST_INST1.uuid,
powervm.TEST_INST1.name, 'data')
mock_run.assert_called_with('upload', 'powervm_nvram',
mock.ANY, options=None)
self.assertEqual(mock_run.call_count, 2)
@mock.patch('nova_powervm.virt.powervm.nvram.swift.SwiftNvramStore.'
'_exists')
@@ -157,13 +164,14 @@ class TestSwiftStore(test.TestCase):
options={'leave_segments': True})
# Test retry upload
mock_run.side_effect = [swft_exc.ClientException('Error message.'),
self._build_results(['obj'])]
mock_res_obj = [{'success': False,
'error': swft_exc.
ClientException('Error Message.')}]
mock_run.side_effect = [mock_res_obj, self._build_results(['obj'])]
self.swift_store._store(powervm.TEST_INST1.uuid,
powervm.TEST_INST1.name, 'data')
mock_run.assert_called_with(
'upload', 'powervm_nvram', mock.ANY,
options={'leave_segments': True})
mock_run.assert_called_with('upload', 'powervm_nvram', mock.ANY,
options={'leave_segments': True})
@mock.patch('nova_powervm.virt.powervm.nvram.swift.SwiftNvramStore.'
'_exists')

52
nova_powervm/virt/powervm/nvram/swift.py
View File

@@ -152,31 +152,45 @@ class SwiftNvramStore(api.NvramStore):
# implement tell/see/reset. If the authentication error occurs during
# upload, this ClientException is raised with no retry. For any other
# operation, swift client will retry and succeed.
@retrying.retry(retry_on_result=lambda result: not result,
@retrying.retry(retry_on_result=lambda result: result,
wait_fixed=250, stop_max_attempt_number=2)
def _run_upload_operation():
try:
return self._run_operation('upload', self.container,
[obj], options=options)
except swft_exc.ClientException:
# Upload operation failed due to expired Keystone token.
# Retry SwiftClient operation to allow regeneration of token.
return None
"""Run the upload operation
Attempts retry for a maximum number of two times. The upload
operation will fail with ClientException, if there is an
authentication error. The second attempt only happens if the
first attempt failed with ClientException. A return value of
True means we should retry, and False means no failure during
upload, thus no retry is required.
Raises RetryError if the upload failed during second attempt,
as the number of attempts for retry is reached.
"""
results = self._run_operation('upload', self.container,
[obj], options=options)
for result in results:
if not result['success']:
# TODO(arun-mani - Bug 1611011): Filed for updating swift
# client to return http status code in case of failure
if isinstance(result['error'], swft_exc.ClientException):
# Upload operation failed due to expired Keystone
# token. Retry SwiftClient operation to allow
# regeneration of token.
return True
# The upload failed.
raise api.NVRAMUploadException(instance=inst_name,
reason=result)
return False
try:
results = _run_upload_operation()
_run_upload_operation()
except retrying.RetryError as re:
# The upload failed.
reason = (_('Unable to store NVRAM after %d attempts') %
re.last_attempt.attempt_number)
raise api.NVRAMUploadException(instance=inst_name, reason=reason)
for result in results:
if not result['success']:
# The upload failed.
raise api.NVRAMUploadException(instance=inst_name,
reason=result)
@lockutils.synchronized('nvram')
def store(self, instance, data, force=True):
"""Store the NVRAM into the storage service.
@@ -275,8 +289,8 @@ class SwiftNvramStore(api.NvramStore):
:param inst_key: The instance key to use for the storage operation.
"""
for result in self._run_operation(
'delete', container=self.container, objects=[inst_key]):
for result in self._run_operation('delete', container=self.container,
objects=[inst_key]):
LOG.debug('Delete slot map result: %s' % str(result))
if not result['success']:
@@ -288,8 +302,8 @@ class SwiftNvramStore(api.NvramStore):
:param instance: instance object
"""
for result in self._run_operation(
'delete', container=self.container, objects=[instance.uuid]):
for result in self._run_operation('delete', container=self.container,
objects=[instance.uuid]):
LOG.debug('Delete result: %s' % str(result), instance=instance)
if not result['success']: