Support pagination for 'list' API calls

Fix 'list' CLI calls when API calls are paginated.
Fix some UUID or name lookups that failed when the number of resources
exceeded 'pagination_max_limit'.

Story 2008892
Task 42489

Change-Id: Ica7c8c953c11af4400b224ad942023f69a72ae02
(cherry picked from commit cf0c4be37e)
This commit is contained in:
Gregory Thiemonge 2021-05-19 11:52:41 +02:00
parent fd7cd86832
commit ea0eedc54a
4 changed files with 121 additions and 42 deletions

View File

@ -11,54 +11,72 @@
# under the License. # under the License.
# #
LOADBALANCER_RESOURCES = 'loadbalancers'
LISTENER_RESOURCES = 'listeners'
POOL_RESOURCES = 'pools'
MEMBER_RESOURCES = 'members'
HEALTH_MONITOR_RESOURCES = 'healthmonitors'
L7POLICY_RESOURCES = 'l7policies'
L7RULE_RESOURCES = 'rules'
QUOTA_RESOURCES = 'quotas'
AMPHORA_RESOURCES = 'amphorae'
PROVIDER_RESOURCES = 'providers'
PROVIDER_FLAVOR_CAPABILITY_RESOURCES = 'flavor_capabilities'
PROVIDER_AVAILABILITY_ZONE_CAPABILITY_RESOURCES = (
'availability_zone_capabilities')
FLAVOR_RESOURCES = 'flavors'
FLAVORPROFILE_RESOURCES = 'flavorprofiles'
AVAILABILITYZONE_RESOURCES = 'availability_zones'
AVAILABILITYZONEPROFILE_RESOURCES = 'availability_zone_profiles'
BASE_LBAAS_ENDPOINT = '/lbaas' BASE_LBAAS_ENDPOINT = '/lbaas'
BASE_OCTAVIA_ENDPOINT = '/octavia' BASE_OCTAVIA_ENDPOINT = '/octavia'
BASE_LOADBALANCER_URL = BASE_LBAAS_ENDPOINT + '/loadbalancers' BASE_LOADBALANCER_URL = BASE_LBAAS_ENDPOINT + '/' + LOADBALANCER_RESOURCES
BASE_SINGLE_LB_URL = BASE_LOADBALANCER_URL + '/{uuid}' BASE_SINGLE_LB_URL = BASE_LOADBALANCER_URL + '/{uuid}'
BASE_LB_STATS_URL = BASE_SINGLE_LB_URL + '/stats' BASE_LB_STATS_URL = BASE_SINGLE_LB_URL + '/stats'
BASE_LOADBALANCER_STATUS_URL = BASE_SINGLE_LB_URL + '/status' BASE_LOADBALANCER_STATUS_URL = BASE_SINGLE_LB_URL + '/status'
BASE_LOADBALANCER_FAILOVER_URL = BASE_SINGLE_LB_URL + '/failover' BASE_LOADBALANCER_FAILOVER_URL = BASE_SINGLE_LB_URL + '/failover'
BASE_LISTENER_URL = BASE_LBAAS_ENDPOINT + '/listeners' BASE_LISTENER_URL = BASE_LBAAS_ENDPOINT + '/' + LISTENER_RESOURCES
BASE_SINGLE_LISTENER_URL = BASE_LISTENER_URL + '/{uuid}' BASE_SINGLE_LISTENER_URL = BASE_LISTENER_URL + '/{uuid}'
BASE_LISTENER_STATS_URL = BASE_SINGLE_LISTENER_URL + '/stats' BASE_LISTENER_STATS_URL = BASE_SINGLE_LISTENER_URL + '/stats'
BASE_POOL_URL = BASE_LBAAS_ENDPOINT + '/pools' BASE_POOL_URL = BASE_LBAAS_ENDPOINT + '/' + POOL_RESOURCES
BASE_SINGLE_POOL_URL = BASE_POOL_URL + '/{pool_id}' BASE_SINGLE_POOL_URL = BASE_POOL_URL + '/{pool_id}'
BASE_MEMBER_URL = BASE_SINGLE_POOL_URL + '/members' BASE_MEMBER_URL = BASE_SINGLE_POOL_URL + '/' + MEMBER_RESOURCES
BASE_SINGLE_MEMBER_URL = BASE_MEMBER_URL + '/{member_id}' BASE_SINGLE_MEMBER_URL = BASE_MEMBER_URL + '/{member_id}'
BASE_HEALTH_MONITOR_URL = BASE_LBAAS_ENDPOINT + '/healthmonitors' BASE_HEALTH_MONITOR_URL = BASE_LBAAS_ENDPOINT + '/' + HEALTH_MONITOR_RESOURCES
BASE_SINGLE_HEALTH_MONITOR_URL = BASE_HEALTH_MONITOR_URL + '/{uuid}' BASE_SINGLE_HEALTH_MONITOR_URL = BASE_HEALTH_MONITOR_URL + '/{uuid}'
BASE_L7POLICY_URL = BASE_LBAAS_ENDPOINT + '/l7policies' BASE_L7POLICY_URL = BASE_LBAAS_ENDPOINT + '/' + L7POLICY_RESOURCES
BASE_SINGLE_L7POLICY_URL = BASE_L7POLICY_URL + '/{policy_uuid}' BASE_SINGLE_L7POLICY_URL = BASE_L7POLICY_URL + '/{policy_uuid}'
BASE_L7RULE_URL = BASE_SINGLE_L7POLICY_URL + '/rules' BASE_L7RULE_URL = BASE_SINGLE_L7POLICY_URL + '/' + L7RULE_RESOURCES
BASE_SINGLE_L7RULE_URL = BASE_SINGLE_L7POLICY_URL + '/rules/{rule_uuid}' BASE_SINGLE_L7RULE_URL = BASE_L7RULE_URL + '/{rule_uuid}'
BASE_QUOTA_URL = BASE_LBAAS_ENDPOINT + '/quotas' BASE_QUOTA_URL = BASE_LBAAS_ENDPOINT + '/' + QUOTA_RESOURCES
BASE_SINGLE_QUOTA_URL = BASE_QUOTA_URL + '/{uuid}' BASE_SINGLE_QUOTA_URL = BASE_QUOTA_URL + '/{uuid}'
BASE_QUOTA_DEFAULT_URL = BASE_QUOTA_URL + '/defaults' BASE_QUOTA_DEFAULT_URL = BASE_QUOTA_URL + '/defaults'
BASE_AMPHORA_URL = BASE_OCTAVIA_ENDPOINT + "/amphorae" BASE_AMPHORA_URL = BASE_OCTAVIA_ENDPOINT + "/" + AMPHORA_RESOURCES
BASE_SINGLE_AMPHORA_URL = BASE_AMPHORA_URL + "/{uuid}" BASE_SINGLE_AMPHORA_URL = BASE_AMPHORA_URL + "/{uuid}"
BASE_AMPHORA_CONFIGURE_URL = BASE_SINGLE_AMPHORA_URL + '/config' BASE_AMPHORA_CONFIGURE_URL = BASE_SINGLE_AMPHORA_URL + '/config'
BASE_AMPHORA_FAILOVER_URL = BASE_SINGLE_AMPHORA_URL + '/failover' BASE_AMPHORA_FAILOVER_URL = BASE_SINGLE_AMPHORA_URL + '/failover'
BASE_AMPHORA_STATS_URL = BASE_SINGLE_AMPHORA_URL + '/stats' BASE_AMPHORA_STATS_URL = BASE_SINGLE_AMPHORA_URL + '/stats'
BASE_PROVIDER_URL = BASE_LBAAS_ENDPOINT + "/providers" BASE_PROVIDER_URL = BASE_LBAAS_ENDPOINT + "/" + PROVIDER_RESOURCES
BASE_PROVIDER_FLAVOR_CAPABILITY_URL = ( BASE_PROVIDER_FLAVOR_CAPABILITY_URL = (
BASE_LBAAS_ENDPOINT + "/providers/{provider}/flavor_capabilities") BASE_PROVIDER_URL + "/{provider}/" + PROVIDER_FLAVOR_CAPABILITY_RESOURCES)
BASE_PROVIDER_AVAILABILITY_ZONE_CAPABILITY_URL = ( BASE_PROVIDER_AVAILABILITY_ZONE_CAPABILITY_URL = (
BASE_LBAAS_ENDPOINT + "/providers/{provider}" BASE_PROVIDER_URL + "/{provider}/" +
"/availability_zone_capabilities" PROVIDER_AVAILABILITY_ZONE_CAPABILITY_RESOURCES
) )
BASE_FLAVOR_URL = BASE_LBAAS_ENDPOINT + "/flavors" BASE_FLAVOR_URL = BASE_LBAAS_ENDPOINT + "/" + FLAVOR_RESOURCES
BASE_SINGLE_FLAVOR_URL = BASE_FLAVOR_URL + "/{uuid}" BASE_SINGLE_FLAVOR_URL = BASE_FLAVOR_URL + "/{uuid}"
BASE_FLAVORPROFILE_URL = BASE_LBAAS_ENDPOINT + "/flavorprofiles" BASE_FLAVORPROFILE_URL = BASE_LBAAS_ENDPOINT + "/" + FLAVORPROFILE_RESOURCES
BASE_SINGLE_FLAVORPROFILE_URL = BASE_FLAVORPROFILE_URL + "/{uuid}" BASE_SINGLE_FLAVORPROFILE_URL = BASE_FLAVORPROFILE_URL + "/{uuid}"
BASE_AVAILABILITYZONE_URL = BASE_LBAAS_ENDPOINT + "/availabilityzones" BASE_AVAILABILITYZONE_URL = BASE_LBAAS_ENDPOINT + "/availabilityzones"

View File

@ -12,6 +12,7 @@
# #
"""Octavia API Library""" """Octavia API Library"""
import functools import functools
import urllib.parse as urlparse
from osc_lib.api import api from osc_lib.api import api
from osc_lib import exceptions as osc_exc from osc_lib import exceptions as osc_exc
@ -66,17 +67,39 @@ class OctaviaAPI(api.BaseAPI):
_endpoint_suffix = '/v2.0' _endpoint_suffix = '/v2.0'
# Make sure we are always requesting JSON responses
JSON_HEADER = {'Accept': 'application/json'}
def __init__(self, endpoint=None, **kwargs): def __init__(self, endpoint=None, **kwargs):
super().__init__(endpoint=endpoint, **kwargs) super().__init__(endpoint=endpoint, **kwargs)
self.endpoint = self.endpoint.rstrip('/') self.endpoint = self.endpoint.rstrip('/')
self._build_url() self._build_url()
# Make sure we are always requesting JSON responses self._create = functools.partial(self.create, headers=self.JSON_HEADER)
JSON_HEADER = {'Accept': 'application/json'} self._delete = functools.partial(self.delete, headers=self.JSON_HEADER)
self._create = functools.partial(self.create, headers=JSON_HEADER) self._find = functools.partial(self.find, headers=self.JSON_HEADER)
self._delete = functools.partial(self.delete, headers=JSON_HEADER)
self._find = functools.partial(self.find, headers=JSON_HEADER) def _list(self, path, **params):
self._list = functools.partial(self.list, headers=JSON_HEADER) get_all = params.pop('get_all', False)
if not get_all:
return self.list(path, **params, headers=self.JSON_HEADER)
# Enable pagination for 'resources'
resource_key = params.pop('resources')
res = []
while True:
response = self.list(path, **params, headers=self.JSON_HEADER)
res.extend(response[resource_key])
links = response.get("{}_links".format(resource_key), [])
for link in links:
if link.get('rel') == 'next':
query_str = urlparse.urlparse(link['href']).query
params = urlparse.parse_qs(query_str)
break
else:
break
return {resource_key: res}
def _build_url(self): def _build_url(self):
if not self.endpoint.endswith(self._endpoint_suffix): if not self.endpoint.endswith(self._endpoint_suffix):
@ -91,7 +114,9 @@ class OctaviaAPI(api.BaseAPI):
List of load balancers List of load balancers
""" """
url = const.BASE_LOADBALANCER_URL url = const.BASE_LOADBALANCER_URL
response = self._list(url, **params) response = self._list(url, get_all=True,
resources=const.LOADBALANCER_RESOURCES,
**params)
return response return response
@ -203,7 +228,9 @@ class OctaviaAPI(api.BaseAPI):
List of listeners List of listeners
""" """
url = const.BASE_LISTENER_URL url = const.BASE_LISTENER_URL
response = self._list(url, **kwargs) response = self._list(url, get_all=True,
resources=const.LISTENER_RESOURCES,
**kwargs)
return response return response
@ -286,7 +313,9 @@ class OctaviaAPI(api.BaseAPI):
List of pools List of pools
""" """
url = const.BASE_POOL_URL url = const.BASE_POOL_URL
response = self._list(url, **kwargs) response = self._list(url, get_all=True,
resources=const.POOL_RESOURCES,
**kwargs)
return response return response
@ -358,7 +387,9 @@ class OctaviaAPI(api.BaseAPI):
Response list members Response list members
""" """
url = const.BASE_MEMBER_URL.format(pool_id=pool_id) url = const.BASE_MEMBER_URL.format(pool_id=pool_id)
response = self._list(url, **kwargs) response = self._list(url, get_all=True,
resources=const.MEMBER_RESOURCES,
**kwargs)
return response return response
@ -441,7 +472,9 @@ class OctaviaAPI(api.BaseAPI):
List of l7policies List of l7policies
""" """
url = const.BASE_L7POLICY_URL url = const.BASE_L7POLICY_URL
response = self._list(url, **kwargs) response = self._list(url, get_all=True,
resources=const.L7POLICY_RESOURCES,
**kwargs)
return response return response
@ -511,7 +544,9 @@ class OctaviaAPI(api.BaseAPI):
List of l7rules List of l7rules
""" """
url = const.BASE_L7RULE_URL.format(policy_uuid=l7policy_id) url = const.BASE_L7RULE_URL.format(policy_uuid=l7policy_id)
response = self._list(url, **kwargs) response = self._list(url, get_all=True,
resources=const.L7RULE_RESOURCES,
**kwargs)
return response return response
@ -592,7 +627,9 @@ class OctaviaAPI(api.BaseAPI):
A dict containing a list of health monitors A dict containing a list of health monitors
""" """
url = const.BASE_HEALTH_MONITOR_URL url = const.BASE_HEALTH_MONITOR_URL
response = self._list(url, **kwargs) response = self._list(url, get_all=True,
resources=const.HEALTH_MONITOR_RESOURCES,
**kwargs)
return response return response
@ -665,7 +702,9 @@ class OctaviaAPI(api.BaseAPI):
A ``dict`` representing a list of quotas for the project A ``dict`` representing a list of quotas for the project
""" """
url = const.BASE_QUOTA_URL url = const.BASE_QUOTA_URL
response = self._list(url, **params) response = self._list(url, get_all=True,
resources=const.QUOTA_RESOURCES,
**params)
return response return response
@ -746,7 +785,9 @@ class OctaviaAPI(api.BaseAPI):
A ``dict`` containing a list of amphorae A ``dict`` containing a list of amphorae
""" """
url = const.BASE_AMPHORA_URL url = const.BASE_AMPHORA_URL
response = self._list(path=url, **kwargs) response = self._list(path=url, get_all=True,
resources=const.AMPHORA_RESOURCES,
**kwargs)
return response return response
@ -812,7 +853,8 @@ class OctaviaAPI(api.BaseAPI):
A ``dict`` containing a list of provider A ``dict`` containing a list of provider
""" """
url = const.BASE_PROVIDER_URL url = const.BASE_PROVIDER_URL
response = self._list(path=url) response = self._list(path=url, get_all=True,
resources=const.PROVIDER_RESOURCES)
return response return response
@ -826,7 +868,9 @@ class OctaviaAPI(api.BaseAPI):
""" """
url = const.BASE_PROVIDER_FLAVOR_CAPABILITY_URL.format( url = const.BASE_PROVIDER_FLAVOR_CAPABILITY_URL.format(
provider=provider) provider=provider)
response = self._list(url) resources = const.PROVIDER_FLAVOR_CAPABILITY_RESOURCES
response = self._list(url, get_all=True,
resources=resources)
return response return response
@ -840,7 +884,9 @@ class OctaviaAPI(api.BaseAPI):
""" """
url = const.BASE_PROVIDER_AVAILABILITY_ZONE_CAPABILITY_URL.format( url = const.BASE_PROVIDER_AVAILABILITY_ZONE_CAPABILITY_URL.format(
provider=provider) provider=provider)
response = self._list(url) resources = const.PROVIDER_AVAILABILITY_ZONE_CAPABILITY_RESOURCES
response = self._list(url, get_all=True,
resources=resources)
return response return response
@ -853,7 +899,8 @@ class OctaviaAPI(api.BaseAPI):
A ``dict`` containing a list of flavor A ``dict`` containing a list of flavor
""" """
url = const.BASE_FLAVOR_URL url = const.BASE_FLAVOR_URL
response = self._list(path=url, **kwargs) response = self._list(path=url, get_all=True,
resources=const.FLAVOR_RESOURCES, **kwargs)
return response return response
@ -937,7 +984,9 @@ class OctaviaAPI(api.BaseAPI):
List of flavor profile List of flavor profile
""" """
url = const.BASE_FLAVORPROFILE_URL url = const.BASE_FLAVORPROFILE_URL
response = self._list(url, **kwargs) response = self._list(url, get_all=True,
resources=const.FLAVORPROFILE_RESOURCES,
**kwargs)
return response return response
@ -993,7 +1042,9 @@ class OctaviaAPI(api.BaseAPI):
A ``dict`` containing a list of availabilityzone A ``dict`` containing a list of availabilityzone
""" """
url = const.BASE_AVAILABILITYZONE_URL url = const.BASE_AVAILABILITYZONE_URL
response = self._list(path=url, **kwargs) response = self._list(path=url, get_all=True,
resources=const.AVAILABILITYZONE_RESOURCES,
**kwargs)
return response return response
@ -1081,7 +1132,10 @@ class OctaviaAPI(api.BaseAPI):
List of availabilityzone profile List of availabilityzone profile
""" """
url = const.BASE_AVAILABILITYZONEPROFILE_URL url = const.BASE_AVAILABILITYZONEPROFILE_URL
response = self._list(url, **kwargs) resources = const.AVAILABILITYZONEPROFILE_RESOURCES
response = self._list(url, get_all=True,
resources=resources,
**kwargs)
return response return response

View File

@ -68,10 +68,11 @@ LIST_ME_RESP = {
{'name': 'mem2'}] {'name': 'mem2'}]
} }
LIST_L7PO_RESP = [ LIST_L7PO_RESP = {
{'name': 'l71'}, 'l7policies':
{'name': 'l72'}, [{'name': 'l71'},
] {'name': 'l72'}]
}
LIST_L7RU_RESP = { LIST_L7RU_RESP = {
'rules': 'rules':

View File

@ -0,0 +1,6 @@
---
fixes:
- |
Fixed some issues when the number of Octavia resources exceeded the
'pagination_max_limit' parameter in Octavia API. The list calls now support
pagination.