Retry to fetch paginated volumes if we get 404 for next link
When we get a volume list, it's possible for a volume to disappear causing the pagination to bork. When that happens we retry to get the list from scratch for 5 times. If all the attempts fail we just return what we found. Change-Id: Ia88d1d8a6b3558f7d5d364a9c78bbf834836d3f7 Signed-off-by: Rosario Di Somma <rosario.disomma@dreamhost.com>
This commit is contained in:
parent
77b2a441cf
commit
6b325282d3
|
@ -1700,21 +1700,53 @@ class OpenStackCloud(
|
||||||
|
|
||||||
"""
|
"""
|
||||||
def _list(data):
|
def _list(data):
|
||||||
volumes.extend(meta.get_and_munchify('volumes', data))
|
volumes.extend(data.get('volumes', []))
|
||||||
endpoint = None
|
endpoint = None
|
||||||
for l in data.get('volumes_links', []):
|
for l in data.get('volumes_links', []):
|
||||||
if 'rel' in l and 'next' == l['rel']:
|
if 'rel' in l and 'next' == l['rel']:
|
||||||
endpoint = l['href']
|
endpoint = l['href']
|
||||||
break
|
break
|
||||||
if endpoint:
|
if endpoint:
|
||||||
_list(self._volume_client.get(endpoint))
|
try:
|
||||||
|
_list(self._volume_client.get(endpoint))
|
||||||
|
except OpenStackCloudURINotFound:
|
||||||
|
# Catch and re-raise here because we are making recursive
|
||||||
|
# calls and we just have context for the log here
|
||||||
|
self.log.debug(
|
||||||
|
"While listing volumes, could not find next link"
|
||||||
|
" {link}.".format(link=data))
|
||||||
|
raise
|
||||||
|
|
||||||
if not cache:
|
if not cache:
|
||||||
warnings.warn('cache argument to list_volumes is deprecated. Use '
|
warnings.warn('cache argument to list_volumes is deprecated. Use '
|
||||||
'invalidate instead.')
|
'invalidate instead.')
|
||||||
volumes = []
|
|
||||||
_list(self._volume_client.get('/volumes/detail'))
|
# Fetching paginated volumes can fails for several reasons, if
|
||||||
return self._normalize_volumes(volumes)
|
# something goes wrong we'll have to start fetching volumes from
|
||||||
|
# scratch
|
||||||
|
attempts = 5
|
||||||
|
for _ in range(attempts):
|
||||||
|
volumes = []
|
||||||
|
data = self._volume_client.get('/volumes/detail')
|
||||||
|
if 'volumes_links' not in data:
|
||||||
|
# no pagination needed
|
||||||
|
volumes.extend(data.get('volumes', []))
|
||||||
|
break
|
||||||
|
|
||||||
|
try:
|
||||||
|
_list(data)
|
||||||
|
break
|
||||||
|
except OpenStackCloudURINotFound:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
self.log.debug(
|
||||||
|
"List volumes failed to retrieve all volumes after"
|
||||||
|
" {attempts} attempts. Returning what we found.".format(
|
||||||
|
attempts=attempts))
|
||||||
|
# list volumes didn't complete succesfully so just return what
|
||||||
|
# we found
|
||||||
|
return self._normalize_volumes(
|
||||||
|
meta.get_and_munchify(key=None, data=volumes))
|
||||||
|
|
||||||
@_utils.cache_on_arguments()
|
@_utils.cache_on_arguments()
|
||||||
def list_volume_types(self, get_extra=True):
|
def list_volume_types(self, get_extra=True):
|
||||||
|
|
|
@ -319,3 +319,96 @@ class TestVolume(base.RequestsMockTestCase):
|
||||||
self.cloud._normalize_volume(vol2)],
|
self.cloud._normalize_volume(vol2)],
|
||||||
self.cloud.list_volumes())
|
self.cloud.list_volumes())
|
||||||
self.assert_calls()
|
self.assert_calls()
|
||||||
|
|
||||||
|
def test_list_volumes_with_pagination_next_link_fails_once(self):
|
||||||
|
vol1 = meta.obj_to_munch(fakes.FakeVolume('01', 'available', 'vol1'))
|
||||||
|
vol2 = meta.obj_to_munch(fakes.FakeVolume('02', 'available', 'vol2'))
|
||||||
|
self.register_uris([
|
||||||
|
dict(method='GET',
|
||||||
|
uri=self.get_mock_url(
|
||||||
|
'volumev2', 'public',
|
||||||
|
append=['volumes', 'detail']),
|
||||||
|
json={
|
||||||
|
'volumes': [vol1],
|
||||||
|
'volumes_links': [
|
||||||
|
{'href': self.get_mock_url(
|
||||||
|
'volumev2', 'public',
|
||||||
|
append=['volumes', 'detail'],
|
||||||
|
qs_elements=['marker=01']),
|
||||||
|
'rel': 'next'}]}),
|
||||||
|
dict(method='GET',
|
||||||
|
uri=self.get_mock_url(
|
||||||
|
'volumev2', 'public',
|
||||||
|
append=['volumes', 'detail'],
|
||||||
|
qs_elements=['marker=01']),
|
||||||
|
status_code=404),
|
||||||
|
dict(method='GET',
|
||||||
|
uri=self.get_mock_url(
|
||||||
|
'volumev2', 'public',
|
||||||
|
append=['volumes', 'detail']),
|
||||||
|
json={
|
||||||
|
'volumes': [vol1],
|
||||||
|
'volumes_links': [
|
||||||
|
{'href': self.get_mock_url(
|
||||||
|
'volumev2', 'public',
|
||||||
|
append=['volumes', 'detail'],
|
||||||
|
qs_elements=['marker=01']),
|
||||||
|
'rel': 'next'}]}),
|
||||||
|
dict(method='GET',
|
||||||
|
uri=self.get_mock_url(
|
||||||
|
'volumev2', 'public',
|
||||||
|
append=['volumes', 'detail'],
|
||||||
|
qs_elements=['marker=01']),
|
||||||
|
json={
|
||||||
|
'volumes': [vol2],
|
||||||
|
'volumes_links': [
|
||||||
|
{'href': self.get_mock_url(
|
||||||
|
'volumev2', 'public',
|
||||||
|
append=['volumes', 'detail'],
|
||||||
|
qs_elements=['marker=02']),
|
||||||
|
'rel': 'next'}]}),
|
||||||
|
|
||||||
|
dict(method='GET',
|
||||||
|
uri=self.get_mock_url(
|
||||||
|
'volumev2', 'public',
|
||||||
|
append=['volumes', 'detail'],
|
||||||
|
qs_elements=['marker=02']),
|
||||||
|
json={'volumes': []})])
|
||||||
|
self.assertEqual(
|
||||||
|
[self.cloud._normalize_volume(vol1),
|
||||||
|
self.cloud._normalize_volume(vol2)],
|
||||||
|
self.cloud.list_volumes())
|
||||||
|
self.assert_calls()
|
||||||
|
|
||||||
|
def test_list_volumes_with_pagination_next_link_fails_all_attempts(self):
|
||||||
|
vol1 = meta.obj_to_munch(fakes.FakeVolume('01', 'available', 'vol1'))
|
||||||
|
uris = []
|
||||||
|
attempts = 5
|
||||||
|
for i in range(attempts):
|
||||||
|
uris.extend([
|
||||||
|
dict(method='GET',
|
||||||
|
uri=self.get_mock_url(
|
||||||
|
'volumev2', 'public',
|
||||||
|
append=['volumes', 'detail']),
|
||||||
|
json={
|
||||||
|
'volumes': [vol1],
|
||||||
|
'volumes_links': [
|
||||||
|
{'href': self.get_mock_url(
|
||||||
|
'volumev2', 'public',
|
||||||
|
append=['volumes', 'detail'],
|
||||||
|
qs_elements=['marker=01']),
|
||||||
|
'rel': 'next'}]}),
|
||||||
|
dict(method='GET',
|
||||||
|
uri=self.get_mock_url(
|
||||||
|
'volumev2', 'public',
|
||||||
|
append=['volumes', 'detail'],
|
||||||
|
qs_elements=['marker=01']),
|
||||||
|
status_code=404)])
|
||||||
|
self.register_uris(uris)
|
||||||
|
# Check that found volumes are returned even if pagination didn't
|
||||||
|
# complete because call to get next link 404'ed for all the allowed
|
||||||
|
# attempts
|
||||||
|
self.assertEqual(
|
||||||
|
[self.cloud._normalize_volume(vol1)],
|
||||||
|
self.cloud.list_volumes())
|
||||||
|
self.assert_calls()
|
||||||
|
|
Loading…
Reference in New Issue