openstack/nova-powervm
Code Issues Proposed changes

Merge "Deploy of a VM occasionally fails due to invalid keystone token"

Browse Source
This commit is contained in:
Jenkins
2016-08-10 15:11:02 +00:00
committed by Gerrit Code Review
parent e90cc307dd 68d31b0cf6
commit 26c4d7bc5d
2 changed files with 49 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
nova_powervm/tests/virt/powervm/nvram/test_swift.py
View File

@@ -16,6 +16,7 @@
import mock
from nova import test
from requests.exceptions import RequestException
from swiftclient import exceptions as swft_exc
from swiftclient import service as swft_srv
@@ -131,18 +132,24 @@ class TestSwiftStore(test.TestCase):
mock.ANY, options=None)
# Test unsuccessful upload
mock_run.return_value[0]['success'] = False
mock_result = [{'success': False,
'error': RequestException('Error Message.')}]
mock_run.return_value = mock_result
self.assertRaises(api.NVRAMUploadException,
self.swift_store._store, powervm.TEST_INST1.uuid,
powervm.TEST_INST1.name, 'data')
# Test retry upload
mock_run.side_effect = [swft_exc.ClientException('Error message.'),
self._build_results(['obj'])]
mock_run.reset_mock()
mock_res_obj = [{'success': False,
'error': swft_exc.
ClientException('Error Message.')}]
mock_run.side_effect = [mock_res_obj, self._build_results(['obj'])]
self.swift_store._store(powervm.TEST_INST1.uuid,
powervm.TEST_INST1.name, 'data')
mock_run.assert_called_with('upload', 'powervm_nvram',
mock.ANY, options=None)
self.assertEqual(mock_run.call_count, 2)
@mock.patch('nova_powervm.virt.powervm.nvram.swift.SwiftNvramStore.'
'_exists')
@@ -157,13 +164,14 @@ class TestSwiftStore(test.TestCase):
options={'leave_segments': True})
# Test retry upload
mock_run.side_effect = [swft_exc.ClientException('Error message.'),
self._build_results(['obj'])]
mock_res_obj = [{'success': False,
'error': swft_exc.
ClientException('Error Message.')}]
mock_run.side_effect = [mock_res_obj, self._build_results(['obj'])]
self.swift_store._store(powervm.TEST_INST1.uuid,
powervm.TEST_INST1.name, 'data')
mock_run.assert_called_with(
'upload', 'powervm_nvram', mock.ANY,
options={'leave_segments': True})
mock_run.assert_called_with('upload', 'powervm_nvram', mock.ANY,
options={'leave_segments': True})
@mock.patch('nova_powervm.virt.powervm.nvram.swift.SwiftNvramStore.'
'_exists')

52
nova_powervm/virt/powervm/nvram/swift.py
View File

@@ -152,31 +152,45 @@ class SwiftNvramStore(api.NvramStore):
# implement tell/see/reset. If the authentication error occurs during
# upload, this ClientException is raised with no retry. For any other
# operation, swift client will retry and succeed.
@retrying.retry(retry_on_result=lambda result: not result,
@retrying.retry(retry_on_result=lambda result: result,
wait_fixed=250, stop_max_attempt_number=2)
def _run_upload_operation():
try:
return self._run_operation('upload', self.container,
[obj], options=options)
except swft_exc.ClientException:
# Upload operation failed due to expired Keystone token.
# Retry SwiftClient operation to allow regeneration of token.
return None
"""Run the upload operation
Attempts retry for a maximum number of two times. The upload
operation will fail with ClientException, if there is an
authentication error. The second attempt only happens if the
first attempt failed with ClientException. A return value of
True means we should retry, and False means no failure during
upload, thus no retry is required.
Raises RetryError if the upload failed during second attempt,
as the number of attempts for retry is reached.
"""
results = self._run_operation('upload', self.container,
[obj], options=options)
for result in results:
if not result['success']:
# TODO(arun-mani - Bug 1611011): Filed for updating swift
# client to return http status code in case of failure
if isinstance(result['error'], swft_exc.ClientException):
# Upload operation failed due to expired Keystone
# token. Retry SwiftClient operation to allow
# regeneration of token.
return True
# The upload failed.
raise api.NVRAMUploadException(instance=inst_name,
reason=result)
return False
try:
results = _run_upload_operation()
_run_upload_operation()
except retrying.RetryError as re:
# The upload failed.
reason = (_('Unable to store NVRAM after %d attempts') %
re.last_attempt.attempt_number)
raise api.NVRAMUploadException(instance=inst_name, reason=reason)
for result in results:
if not result['success']:
# The upload failed.
raise api.NVRAMUploadException(instance=inst_name,
reason=result)
@lockutils.synchronized('nvram')
def store(self, instance, data, force=True):
"""Store the NVRAM into the storage service.
@@ -275,8 +289,8 @@ class SwiftNvramStore(api.NvramStore):
:param inst_key: The instance key to use for the storage operation.
"""
for result in self._run_operation(
'delete', container=self.container, objects=[inst_key]):
for result in self._run_operation('delete', container=self.container,
objects=[inst_key]):
LOG.debug('Delete slot map result: %s' % str(result))
if not result['success']:
@@ -288,8 +302,8 @@ class SwiftNvramStore(api.NvramStore):
:param instance: instance object
"""
for result in self._run_operation(
'delete', container=self.container, objects=[instance.uuid]):
for result in self._run_operation('delete', container=self.container,
objects=[instance.uuid]):
LOG.debug('Delete result: %s' % str(result), instance=instance)
if not result['success']: