Support pagination for resource routing list operations

1. What is the problem
The resource routing list operations will retrieve all the items in
the database, which will consume too much memory and take long time
to response when the results are considerably large.

2. What is the solution for the problem
To reduce load on the service, list operations will return a maximum
number of items at a time by pagination. To navigate the collection,
the parameters limit and marker can be set in the URI. For example:

/v1.0/routings?limit=2000&marker=500

The marker parameter is the ID of the last item in the previous list.
A marker with an invalid ID returns a badRequest (400) fault.
The limit parameter sets the page size. These parameters are optional.
If the client requests a limit beyond the maximum limit configured by
the deployment, the server returns the maximum limit number of items.
Pagination and filtering can work together with routing's list operations.

3. What the features need to be implemented to the Tricircle to
realize the solution
Add pagination feature for resource routing list operations.

Change-Id: I05d1b30f502103d247d8be06c1e52fdcec42b41e
This commit is contained in:
Fangming Liu 2017-06-19 11:51:32 +08:00
parent 1b4ceafc44
commit 192aa09d76
8 changed files with 308 additions and 173 deletions

View File

@ -38,10 +38,9 @@ common_opts = [
help=_("Allow the usage of the pagination")),
cfg.BoolOpt('allow_sorting', default=False,
help=_("Allow the usage of the sorting")),
cfg.StrOpt('pagination_max_limit', default="-1",
cfg.IntOpt('pagination_max_limit', min=1, default=2000,
help=_("The maximum number of items returned in a single "
"response, value was 'infinite' or negative integer "
"means no limit")),
"response, value must be greater or equal to 1")),
]

View File

@ -13,7 +13,6 @@
import pecan
from pecan import expose
from pecan import rest
import six
from oslo_log import log as logging
@ -115,21 +114,86 @@ class RoutingController(rest.RestController):
return utils.format_api_error(
403, _('Unauthorized to show all resource routings'))
# default value -1 means no pagination, then maximum pagination
# limit from configuration will be used.
_limit = kwargs.pop('limit', -1)
try:
limit = int(_limit)
limit = utils.get_pagination_limit(limit)
except ValueError as e:
LOG.exception('Failed to convert pagination limit to an integer: '
'%(exception)s ', {'exception': e})
msg = (_("Limit should be an integer or a valid literal "
"for int() rather than '%s'") % _limit)
return utils.format_api_error(400, msg)
marker = kwargs.pop('marker', None)
if marker is not None:
try:
marker = int(marker)
try:
# we throw an exception if a marker with
# invalid ID is specified.
db_api.get_resource_routing(context, marker)
except t_exceptions.ResourceNotFound:
return utils.format_api_error(
400, _('Marker %s is an invalid ID') % marker)
except ValueError as e:
LOG.exception('Failed to convert page marker to an integer: '
'%(exception)s ', {'exception': e})
msg = (_("Marker should be an integer or a valid literal "
"for int() rather than '%s'") % marker)
return utils.format_api_error(400, msg)
if kwargs:
is_valid_filter, filters = self._get_filters(kwargs)
if not is_valid_filter:
msg = (_('Unsupported filter type: %(filters)s') % {
'filters': ', '.join([filter_name for filter_name in filters])
'filters': ', '.join(
[filter_name for filter_name in filters])
})
return utils.format_api_error(400, msg)
filters = [{'key': key,
'comparator': 'eq',
'value': value} for key, value in six.iteritems(filters)]
if 'id' in filters:
try:
# resource routing id is an integer.
filters['id'] = int(filters['id'])
except ValueError as e:
LOG.exception('Failed to convert routing id to an integer:'
' %(exception)s ', {'exception': e})
msg = (_("Id should be an integer or a valid literal "
"for int() rather than '%s'") % filters['id'])
return utils.format_api_error(400, msg)
expand_filters = [{'key': filter_name, 'comparator': 'eq',
'value': filters[filter_name]}
for filter_name in filters]
else:
expand_filters = None
try:
return {'routings': db_api.list_resource_routings(context,
filters)}
routings = db_api.list_resource_routings(context, expand_filters,
limit, marker,
sorts=[('id', 'desc')])
links = []
if len(routings) >= limit:
marker = routings[-1]['id']
# if we reach the first element, then no elements in next page,
# so link to next page won't be provided.
if marker != 1:
base = constants.ROUTING_PATH
link = "%s?limit=%s&marker=%s" % (base, limit, marker)
links.append({"rel": "next",
"href": link})
result = {}
result["routings"] = routings
if links:
result["routings_links"] = links
return result
except Exception as e:
LOG.exception('Failed to show all resource routings: '
'%(exception)s ', {'exception': e})

View File

@ -189,3 +189,6 @@ job_primary_resource_map = {
JT_SFC_SYNC: (RT_PORT_CHAIN, "portchain_id"),
JT_RESOURCE_RECYCLE: (None, "project_id")
}
# resource routing request path
ROUTING_PATH = '/v1.0/routings'

View File

@ -17,6 +17,7 @@ import six
import pecan
from oslo_config import cfg
from oslo_utils import uuidutils
from tricircle.common import constants as cons
@ -165,3 +166,18 @@ def format_nova_error(code, message, error_type=None):
def format_cinder_error(code, message, error_type=None):
return format_error(code, message, error_type)
def get_pagination_limit(_limit):
"""Return page size limitation.
:param _limit: page size from the client.
:return limit: limit sets the page size. If the client requests a limit
beyond the maximum limit in configuration or sets invalid value,
then the maximum limit will be used. If client doesn't set page limit,
maximum pagination limit will be used to control the page size.
"""
max_limit = cfg.CONF.pagination_max_limit
limit = min(_limit, max_limit) if _limit > 0 else max_limit
return limit

View File

@ -112,9 +112,24 @@ def create_resource_mapping(context, top_id, bottom_id, pod_id, project_id,
context.session.close()
def list_resource_routings(context, filters=None, sorts=None):
def list_resource_routings(context, filters=None, limit=None, marker=None,
sorts=None):
"""Return a list of limited number of resource routings
:param context:
:param filters: list of filter dict with key 'key', 'comparator', 'value'
:param limit: an integer that limits the maximum number of items
returned in a single response
:param marker: id of the last item in the previous list
:param sorts: a list of (sort_key, sort_dir) pair,
for example, [('id', 'desc')]
:return: a list of limited number of items
"""
with context.session.begin():
return core.query_resource(context, models.ResourceRouting,
return core.paginate_query(context, models.ResourceRouting,
limit,
models.ResourceRouting(
id=marker) if marker else None,
filters or [], sorts or [])

View File

@ -13,22 +13,23 @@
# License for the specific language governing permissions and limitations
# under the License.
import threading
import sqlalchemy as sql
from sqlalchemy.ext import declarative
from sqlalchemy.inspection import inspect
import threading
from oslo_config import cfg
import oslo_db.options as db_options
import oslo_db.sqlalchemy.session as db_session
from oslo_db.sqlalchemy import utils as sa_utils
from oslo_log import log as logging
from oslo_utils import strutils
from tricircle.common import exceptions
LOG = logging.getLogger(__name__)
db_opts = [
cfg.StrOpt('tricircle_db_connection',
help='db connection string for tricircle'),
@ -87,6 +88,36 @@ def _get_resource(context, model, pk_value):
return res_obj
def paginate_query(context, model, limit, marker_obj, filters, sorts):
"""Returns a query with sorting / pagination / filtering criteria added.
:param context:
:param model:
:param limit: the maximum number of items returned in a single page
:param marker_obj: data model instance that has the same fields as
keys in sorts. All its value(s) are from the last item
of the previous page; we returns the next
results after this item.
:param filters: list of filter dict with key 'key', 'comparator', 'value'
:param sorts: a list of (sort_key, sort_dir) pair,
for example, [('id', 'desc')]
:return: the query with sorting/pagination/filtering added
"""
query = context.session.query(model)
query = _filter_query(model, query, filters)
sort_keys = []
sort_dirs = []
for sort_key, sort_dir in sorts:
sort_keys.append(sort_key)
sort_dirs.append(sort_dir)
query = sa_utils.paginate_query(query, model, limit, marker=marker_obj,
sort_keys=sort_keys, sort_dirs=sort_dirs)
return [obj.to_dict() for obj in query]
def create_resource(context, model, res_dict):
res_obj = model.from_dict(res_dict)
context.session.add(res_obj)

View File

@ -23,7 +23,8 @@ CONFLICT_OPT_NAMES = [
'bind_port',
'bind_host',
'allow_pagination',
'allow_sorting'
'allow_sorting',
'pagination_max_limit',
]

View File

@ -14,10 +14,13 @@ import mock
from mock import patch
from oslo_utils import uuidutils
import six
from six.moves import xrange
import unittest
from oslo_config import cfg
import pecan
from tricircle.api import app
from tricircle.api.controllers import pod
from tricircle.api.controllers import routing
from tricircle.common import context
@ -35,6 +38,8 @@ class FakeResponse(object):
class RoutingControllerTest(unittest.TestCase):
def setUp(self):
cfg.CONF.clear()
cfg.CONF.register_opts(app.common_opts)
core.initialize()
core.ModelBase.metadata.create_all(core.get_engine())
self.controller = routing.RoutingController()
@ -49,45 +54,24 @@ class RoutingControllerTest(unittest.TestCase):
def test_post(self, mock_context):
mock_context.return_value = self.context
# prepare the foreign key: pod_id
kw_pod = {'pod': {'region_name': 'pod1', 'az_name': 'az1'}}
pod_id = pod.PodsController().post(**kw_pod)['pod']['pod_id']
# a variable used for later test
project_id = uuidutils.generate_uuid()
kw_routing = {'routing':
{'top_id': '09fd7cc9-d169-4b5a-88e8-436ecf4d0bfe',
'bottom_id': 'dc80f9de-abb7-4ec6-ab7a-94f8fd1e20ef',
'pod_id': pod_id,
'project_id': project_id,
'resource_type': 'subnet'
}}
kw_routing = self._prepare_routing_element('subnet')
id = self.controller.post(**kw_routing)['routing']['id']
routing = db_api.get_resource_routing(self.context, id)
self.assertEqual('09fd7cc9-d169-4b5a-88e8-436ecf4d0bfe',
routing['top_id'])
self.assertEqual('dc80f9de-abb7-4ec6-ab7a-94f8fd1e20ef',
routing['bottom_id'])
self.assertEqual(pod_id, routing['pod_id'])
self.assertEqual(project_id, routing['project_id'])
self.assertEqual('subnet', routing['resource_type'])
routings = db_api.list_resource_routings(self.context,
[{'key': 'top_id',
[{'key': 'resource_type',
'comparator': 'eq',
'value':
'09fd7cc9-d169-4b5a-'
'88e8-436ecf4d0bfe'
'subnet'
},
{'key': 'pod_id',
'comparator': 'eq',
'value': pod_id}
], [])
])
self.assertEqual(1, len(routings))
# failure case, only admin can create resource routing
self.context.is_admin = False
kw_routing = self._prepare_routing_element('subnet')
res = self.controller.post(**kw_routing)
self._validate_error_code(res, 403)
@ -95,60 +79,39 @@ class RoutingControllerTest(unittest.TestCase):
# failure case, request body not found
kw_routing1 = {'route':
{'top_id': '109fd7cc9-d169-4b5a-88e8-436ecf4d0bfe',
'bottom_id': '2dc80f9de-abb7-4ec6-ab7a-94f8fd1e20ef',
'pod_id': pod_id,
'project_id': project_id,
'resource_type': 'subnet'
{'top_id': uuidutils.generate_uuid(),
'bottom_id': uuidutils.generate_uuid(),
}}
res = self.controller.post(**kw_routing1)
self._validate_error_code(res, 400)
# failure case, top_id is not given
kw_routing2 = {'routing':
{'bottom_id': '2dc80f9de-abb7-4ec6-ab7a-94f8fd1e20ef',
'pod_id': pod_id,
'project_id': project_id,
'resource_type': 'subnet'
}}
kw_routing2 = self._prepare_routing_element('router')
kw_routing2['routing'].pop('top_id')
res = self.controller.post(**kw_routing2)
self._validate_error_code(res, 400)
# failure case, top_id is empty
kw_routing3 = {'routing':
{'top_id': '',
'bottom_id': '2dc80f9de-abb7-4ec6-ab7a-94f8fd1e20ef',
'pod_id': pod_id,
'project_id': project_id,
'resource_type': 'subnet'
}}
kw_routing3 = self._prepare_routing_element('router')
kw_routing3['routing'].update({'top_id': ''})
res = self.controller.post(**kw_routing3)
self._validate_error_code(res, 400)
# failure case, top_id is given value 'None'
kw_routing4 = {'routing':
{'top_id': None,
'bottom_id': '2dc80f9de-abb7-4ec6-ab7a-94f8fd1e20ef',
'pod_id': pod_id,
'project_id': project_id,
'resource_type': 'subnet'
}}
kw_routing4 = self._prepare_routing_element('security_group')
kw_routing4['routing'].update({'top_id': None})
res = self.controller.post(**kw_routing4)
self._validate_error_code(res, 400)
# failure case, wrong resource type
kw_routing6 = {'routing':
{'top_id': '09fd7cc9-d169-4b5a-88e8-436ecf4d0b09',
'bottom_id': 'dc80f9de-abb7-4ec6-ab7a-94f8fd1e2031f',
'pod_id': pod_id,
'project_id': project_id,
'resource_type': 'server'
}}
res = self.controller.post(**kw_routing6)
kw_routing6 = self._prepare_routing_element('server')
self.controller.post(**kw_routing6)
self._validate_error_code(res, 400)
# failure case, the resource routing already exists
res = self.controller.post(**kw_routing)
kw_routing7 = self._prepare_routing_element('router')
self.controller.post(**kw_routing7)
res = self.controller.post(**kw_routing7)
self._validate_error_code(res, 409)
@patch.object(pecan, 'response', new=FakeResponse)
@ -156,30 +119,10 @@ class RoutingControllerTest(unittest.TestCase):
def test_get_one(self, mock_context):
mock_context.return_value = self.context
# prepare the foreign key: pod_id
kw_pod = {'pod': {'region_name': 'pod1', 'az_name': 'az1'}}
pod_id = pod.PodsController().post(**kw_pod)['pod']['pod_id']
# a variable used for later test
project_id = uuidutils.generate_uuid()
kw_routing = {'routing':
{'top_id': '09fd7cc9-d169-4b5a-88e8-436ecf4d0bfe',
'bottom_id': 'dc80f9de-abb7-4ec6-ab7a-94f8fd1e20ef',
'pod_id': pod_id,
'project_id': project_id,
'resource_type': 'port'
}}
kw_routing = self._prepare_routing_element('port')
id = self.controller.post(**kw_routing)['routing']['id']
routing = self.controller.get_one(id)
self.assertEqual('09fd7cc9-d169-4b5a-88e8-436ecf4d0bfe',
routing['routing']['top_id'])
self.assertEqual('dc80f9de-abb7-4ec6-ab7a-94f8fd1e20ef',
routing['routing']['bottom_id'])
self.assertEqual(pod_id, routing['routing']['pod_id'])
self.assertEqual(project_id, routing['routing']['project_id'])
self.assertEqual('port', routing['routing']['resource_type'])
# failure case, only admin can get resource routing
@ -195,74 +138,131 @@ class RoutingControllerTest(unittest.TestCase):
@patch.object(pecan, 'response', new=FakeResponse)
@patch.object(context, 'extract_context_from_environ')
def test_get_all(self, mock_context):
def test_get_routings_with_pagination(self, mock_context):
mock_context.return_value = self.context
# prepare the foreign key: pod_id
kw_pod1 = {'pod': {'region_name': 'pod1', 'az_name': 'az1'}}
pod_id1 = pod.PodsController().post(**kw_pod1)['pod']['pod_id']
# test when no pagination and filters are applied to the list
# operation, then all of the routings will be retrieved.
for resource_type in ('subnet', 'router', 'security_group', 'network'):
kw_routing = self._prepare_routing_element(resource_type)
self.controller.post(**kw_routing)
# a variable used for later test
project_id = uuidutils.generate_uuid()
kw_routing1 = {'routing':
{'top_id': 'c7f641c9-8462-4007-84b2-3035d8cfb7a3',
'bottom_id': 'dc80f9de-abb7-4ec6-ab7a-94f8fd1e20ef',
'pod_id': pod_id1,
'project_id': project_id,
'resource_type': 'subnet'
}}
# prepare the foreign key: pod_id
kw_pod2 = {'pod': {'region_name': 'pod2', 'az_name': 'az1'}}
pod_id2 = pod.PodsController().post(**kw_pod2)['pod']['pod_id']
kw_routing2 = {'routing':
{'top_id': 'b669a2da-ca95-47db-a2a9-ba9e546d82ee',
'bottom_id': 'fd72c010-6e62-4866-b999-6dcb718dd7b4',
'pod_id': pod_id2,
'project_id': project_id,
'resource_type': 'port'
}}
self.controller.post(**kw_routing1)
self.controller.post(**kw_routing2)
# no filters are applied to the routings, so all of the routings will
# be retrieved.
routings = self.controller.get_all()
actual = [(routing['top_id'], routing['pod_id'])
for routing in routings['routings']]
expect = [('c7f641c9-8462-4007-84b2-3035d8cfb7a3', pod_id1),
('b669a2da-ca95-47db-a2a9-ba9e546d82ee', pod_id2)]
six.assertCountEqual(self, expect, actual)
ids = [routing['id']
for key, values in six.iteritems(routings)
for routing in values]
self.assertEqual([4, 3, 2, 1], ids)
# apply a resource type filter to the retrieved routings.
kw_filter1 = {'resource_type': 'port'}
routings = self.controller.get_all(**kw_filter1)
actual = [(routing['top_id'], routing['pod_id'],
routing['resource_type'])
for routing in routings['routings']]
expect = [('b669a2da-ca95-47db-a2a9-ba9e546d82ee', pod_id2, 'port')]
six.assertCountEqual(self, expect, actual)
for filter_name in ('subnet', 'router', 'security_group', 'network'):
filters = {'resource_type': filter_name}
routings = self.controller.get_all(**filters)
items = [routing['resource_type']
for key, values in six.iteritems(routings)
for routing in values]
self.assertEqual(1, len(items))
# test when pagination limit varies in range [1, 5)
for i in xrange(1, 5):
routings = []
total_pages = 0
routing = self.controller.get_all(limit=i)
total_pages += 1
routings.extend(routing['routings'])
while 'routings_links' in routing:
link = routing['routings_links'][0]['href']
_, marker_dict = link.split('&')
# link is like '/v1.0/routings?limit=1&marker=1', after split,
# marker_dict is a string like 'marker=1'.
_, marker_value = marker_dict.split('=')
routing = self.controller.get_all(limit=i, marker=marker_value)
if len(routing['routings']) > 0:
total_pages += 1
routings.extend(routing['routings'])
# assert that total pages will decrease as the limit increase.
pages = int(4 / i)
if 4 % i:
pages += 1
self.assertEqual(pages, total_pages)
self.assertEqual(4, len(routings))
for i in xrange(4):
self.assertEqual(4-i, routings[i]['id'])
set1 = set(['subnet', 'router', 'security_group', 'network'])
set2 = set([routing1['resource_type'] for routing1 in routings])
self.assertEqual(set1, set2)
# test cases when pagination and filters are used
routings = self.controller.get_all(resource_type='network', limit=1)
self.assertEqual(1, len(routings['routings']))
routings = self.controller.get_all(resource_type='subnet', limit=2)
self.assertEqual(1, len(routings['routings']))
# apply a filter and if it doesn't match with any of the retrieved
# routings, then all of them will be discarded and the method returns
# with []
# with [].
kw_filter2 = {'resource_type': 'port2'}
routings = self.controller.get_all(**kw_filter2)
self.assertEqual([], routings['routings'])
# test cases when limit from client is abnormal
routings = self.controller.get_all(limit=0)
self.assertEqual(4, len(routings['routings']))
routings = self.controller.get_all(limit=-1)
self.assertEqual(4, len(routings['routings']))
res = self.controller.get_all(limit='20x')
self._validate_error_code(res, 400)
# test cases when pagination limit from client is greater than
# max limit
pagination_max_limit_backup = cfg.CONF.pagination_max_limit
cfg.CONF.set_override('pagination_max_limit', 2)
routings = self.controller.get_all(limit=3)
self.assertEqual(2, len(routings['routings']))
cfg.CONF.set_override('pagination_max_limit',
pagination_max_limit_backup)
# test case when marker reaches 1, then no link to next page
routings = self.controller.get_all(limit=2, marker=3)
self.assertNotIn('routings_links', routings)
# test cases when marker is abnormal
res = self.controller.get_all(limit=2, marker=-1)
self._validate_error_code(res, 400)
res = self.controller.get_all(limit=2, marker=0)
self._validate_error_code(res, 400)
res = self.controller.get_all(limit=2, marker="last")
self._validate_error_code(res, 400)
# failure case, use an unsupported filter type
kw_filter3 = {'resource': 'port'}
res = self.controller.get_all(**kw_filter3)
self._validate_error_code(res, 400)
kw_filter4 = {'pod_id': pod_id1,
kw_filter4 = {'pod_id': "pod_id_1",
'resource': 'port'}
res = self.controller.get_all(**kw_filter4)
self._validate_error_code(res, 400)
# failure case, id can't be converted to an integer
kw_filter5 = {'id': '4s'}
res = self.controller.get_all(**kw_filter5)
self._validate_error_code(res, 400)
@patch.object(pecan, 'response', new=FakeResponse)
@patch.object(context, 'extract_context_from_environ')
def test_get_all_non_admin(self, mock_context):
mock_context.return_value = self.context
kw_routing1 = self._prepare_routing_element('subnet')
self.controller.post(**kw_routing1)
# failure case, only admin can show all resource routings
self.context.is_admin = False
res = self.controller.get_all()
@ -305,7 +305,7 @@ class RoutingControllerTest(unittest.TestCase):
{'key': 'pod_id',
'comparator': 'eq',
'value': pod_id
}], [])
}])
self.assertEqual(0, len(routings))
# failure case, only admin can delete resource routing
@ -325,24 +325,11 @@ class RoutingControllerTest(unittest.TestCase):
def test_put(self, mock_context):
mock_context.return_value = self.context
# prepare the foreign key: pod_id
kw_pod1 = {'pod': {'region_name': 'pod1', 'az_name': 'az1'}}
pod_id1 = pod.PodsController().post(**kw_pod1)['pod']['pod_id']
# a variable used for later test
project_id = uuidutils.generate_uuid()
body = {'routing':
{'top_id': 'c7f641c9-8462-4007-84b2-3035d8cfb7a3',
'bottom_id': 'dc80f9de-abb7-4ec6-ab7a-94f8fd1e20ef',
'pod_id': pod_id1,
'project_id': project_id,
'resource_type': 'router'
}}
body = self._prepare_routing_element('subnet')
# both bottom_id and resource type have been changed
body_update1 = {'routing':
{'bottom_id': 'fd72c010-6e62-4866-b999-6dcb718dd7b4',
{'bottom_id': uuidutils.generate_uuid(),
'resource_type': 'port'
}}
@ -351,9 +338,8 @@ class RoutingControllerTest(unittest.TestCase):
self.assertEqual('port',
routing['routing']['resource_type'])
self.assertEqual('fd72c010-6e62-4866-b999-6dcb718dd7b4',
self.assertEqual(body_update1['routing']['bottom_id'],
routing['routing']['bottom_id'])
self.assertEqual(pod_id1, routing['routing']['pod_id'])
# failure case, only admin can update resource routing
self.context.is_admin = False
@ -364,7 +350,7 @@ class RoutingControllerTest(unittest.TestCase):
# failure case, request body not found
body_update2 = {'route':
{'bottom_id': 'fd72c010-6e62-4866-b999-6dcb718dd7b4',
{'bottom_id': uuidutils.generate_uuid(),
'resource_type': 'port'
}}
res = self.controller.put(id, **body_update2)
@ -401,5 +387,25 @@ class RoutingControllerTest(unittest.TestCase):
res = self.controller.put(id, **body_update6)
self._validate_error_code(res, 400)
def _prepare_routing_element(self, resource_type):
"""Prepare routing fields except id
:return: A Dictionary with top_id, bottom_id, pod_id,
project_id, resource_type
"""
fake_routing = {
'routing': {
'top_id': uuidutils.generate_uuid(),
'bottom_id': uuidutils.generate_uuid(),
'pod_id': uuidutils.generate_uuid(),
'project_id': uuidutils.generate_uuid(),
'resource_type': resource_type,
}
}
return fake_routing
def tearDown(self):
cfg.CONF.unregister_opts(app.common_opts)
core.ModelBase.metadata.drop_all(core.get_engine())