Implement list limiting support in driver backends
Provides an optional limit to the number of rows that will be returned by a backend from a list_{entity} call. The limit is specified in the configuration file, and allows for an overall general limit as well as an individual limit for a given driver. By default, there is no limit. Limitations: - The list limit is not yet handled by LDAP, rather this remains implemented in the final wrap collection - a subsequent patch will provide the support in the LDAP drivers Implements bp list-limiting Change-Id: I7ca76a8da4260242e578c44103b26257f7e2a5d5
This commit is contained in:
parent
1923a3f5ba
commit
0dec4c8be9
|
@ -694,6 +694,24 @@ should be set to one of the following modes:
|
|||
mechanism called ``named``.
|
||||
|
||||
|
||||
Limiting the number of entities returned in a collection
|
||||
--------------------------------------------------------
|
||||
|
||||
Keystone provides a method of setting a limit to the number of entities
|
||||
returned in a collection, which is useful to prevent overly long response
|
||||
times for list queries that have not specified a sufficently narrow filter.
|
||||
This limit can be set globally by setting ``list_limit`` in the default section
|
||||
of ``keystone.conf``, with no limit set by default. Individual driver
|
||||
sections may override this global value with a specific limit, for example::
|
||||
|
||||
[assignment]
|
||||
list_limit = 100
|
||||
|
||||
If a response to ``list_{entity}`` call has been truncted, then the response
|
||||
status code will still be 200 (OK), but the ``truncated`` attribute in the
|
||||
collection will be set to ``true``.
|
||||
|
||||
|
||||
Sample Configuration Files
|
||||
--------------------------
|
||||
|
||||
|
|
|
@ -144,6 +144,21 @@ The contract for a driver for ``list_{entity}`` methods is therefore:
|
|||
list by filtering for one or more of the specified filters in the passed
|
||||
Hints reference, and removing any such satisfied filters.
|
||||
|
||||
Entity list truncation by drivers
|
||||
---------------------------------
|
||||
|
||||
Keystone supports the ability for a deployment to restrict the number of
|
||||
entries returned from ``list_{entity}`` methods, typically to prevent poorly
|
||||
formed searches (e.g. without sufficient filters) from becoming a performance
|
||||
issue.
|
||||
|
||||
These limits are set in the configuration file, either for a specific driver or
|
||||
across all drivers. These limits are read at the Manager level and passed into
|
||||
individual drivers as part of the Hints list object. A driver should try and
|
||||
honor any such limit if possible, but if it is unable to do so then it may
|
||||
ignore it (and the truncation of the returned list of entities will happen at
|
||||
the controller level).
|
||||
|
||||
Testing
|
||||
-------
|
||||
|
||||
|
|
|
@ -61,6 +61,12 @@
|
|||
# If true, use synchronous mode for sqlite
|
||||
# sqlite_synchronous = True
|
||||
|
||||
# The maximum number of entities that will be returned in a collection can be
|
||||
# set with list_limit, with no limit set by default. This global limit may be
|
||||
# then overridden for a specific driver, by specifying a list_limit in the
|
||||
# appropriate section (e.g. [assignment])
|
||||
# list_limit =
|
||||
|
||||
# === Logging Options ===
|
||||
# Print debugging output
|
||||
# (includes plaintext request logging, potentially including passwords)
|
||||
|
@ -222,6 +228,9 @@
|
|||
# Maximum supported length for user passwords; decrease to improve performance.
|
||||
# max_password_length = 4096
|
||||
|
||||
# Maximum number of entities that will be returned in an identity collection
|
||||
# list_limit =
|
||||
|
||||
[credential]
|
||||
# driver = keystone.credential.backends.sql.Credential
|
||||
|
||||
|
@ -245,6 +254,9 @@
|
|||
|
||||
# template_file = default_catalog.templates
|
||||
|
||||
# Maximum number of entities that will be returned in a catalog collection
|
||||
# list_limit =
|
||||
|
||||
[endpoint_filter]
|
||||
# extension for creating associations between project and endpoints in order to
|
||||
# provide a tailored catalog for project-scoped token requests.
|
||||
|
@ -325,6 +337,9 @@
|
|||
[policy]
|
||||
# driver = keystone.policy.backends.sql.Policy
|
||||
|
||||
# Maximum number of entities that will be returned in a policy collection
|
||||
# list_limit =
|
||||
|
||||
[ec2]
|
||||
# driver = keystone.contrib.ec2.backends.kvs.Ec2
|
||||
|
||||
|
@ -338,6 +353,9 @@
|
|||
# Assignment specific cache time-to-live (TTL) in seconds.
|
||||
# cache_time =
|
||||
|
||||
# Maximum number of entities that will be returned in an assignment collection
|
||||
# list_limit =
|
||||
|
||||
[oauth1]
|
||||
# driver = keystone.contrib.oauth1.backends.sql.OAuth1
|
||||
|
||||
|
|
|
@ -209,10 +209,11 @@ class Assignment(sql.Base, assignment.Driver):
|
|||
self._update_metadata(session, user_id, project_id, metadata_ref,
|
||||
domain_id, group_id)
|
||||
|
||||
@sql.truncated
|
||||
def list_projects(self, hints):
|
||||
with sql.transaction() as session:
|
||||
query = session.query(Project)
|
||||
project_refs = self.filter_query(Project, query, hints)
|
||||
project_refs = self.filter_limit_query(Project, query, hints)
|
||||
return [project_ref.to_dict() for project_ref in project_refs]
|
||||
|
||||
def list_projects_in_domain(self, domain_id):
|
||||
|
@ -528,10 +529,11 @@ class Assignment(sql.Base, assignment.Driver):
|
|||
session.add(ref)
|
||||
return ref.to_dict()
|
||||
|
||||
@sql.truncated
|
||||
def list_domains(self, hints):
|
||||
with sql.transaction() as session:
|
||||
query = session.query(Domain)
|
||||
refs = self.filter_query(Domain, query, hints)
|
||||
refs = self.filter_limit_query(Domain, query, hints)
|
||||
return [ref.to_dict() for ref in refs]
|
||||
|
||||
def _get_domain(self, session, domain_id):
|
||||
|
@ -581,10 +583,11 @@ class Assignment(sql.Base, assignment.Driver):
|
|||
session.add(ref)
|
||||
return ref.to_dict()
|
||||
|
||||
@sql.truncated
|
||||
def list_roles(self, hints):
|
||||
with sql.transaction() as session:
|
||||
query = session.query(Role)
|
||||
refs = self.filter_query(Role, query, hints)
|
||||
refs = self.filter_limit_query(Role, query, hints)
|
||||
return [ref.to_dict() for ref in refs]
|
||||
|
||||
def _get_role(self, session, role_id):
|
||||
|
|
|
@ -266,6 +266,8 @@ class Manager(manager.Manager):
|
|||
"exist."),
|
||||
role_id)
|
||||
|
||||
# TODO(henry-nash): We might want to consider list limiting this at some
|
||||
# point in the future.
|
||||
def list_projects_for_user(self, user_id, hints=None):
|
||||
# NOTE(henry-nash): In order to get a complete list of user projects,
|
||||
# the driver will need to look at group assignments. To avoid cross
|
||||
|
@ -296,6 +298,7 @@ class Manager(manager.Manager):
|
|||
self.get_domain_by_name.set(ret, self, ret['name'])
|
||||
return ret
|
||||
|
||||
@manager.response_truncated
|
||||
def list_domains(self, hints=None):
|
||||
return self.driver.list_domains(hints or driver_hints.Hints())
|
||||
|
||||
|
@ -392,6 +395,7 @@ class Manager(manager.Manager):
|
|||
{'userid': user['id'],
|
||||
'domainid': domain_id})
|
||||
|
||||
@manager.response_truncated
|
||||
def list_projects(self, hints=None):
|
||||
return self.driver.list_projects(hints or driver_hints.Hints())
|
||||
|
||||
|
@ -427,6 +431,7 @@ class Manager(manager.Manager):
|
|||
self.get_role.set(ret, self, role_id)
|
||||
return ret
|
||||
|
||||
@manager.response_truncated
|
||||
def list_roles(self, hints=None):
|
||||
return self.driver.list_roles(hints or driver_hints.Hints())
|
||||
|
||||
|
@ -590,6 +595,9 @@ class Driver(object):
|
|||
inherited).items()))
|
||||
return [dict(r) for r in role_set]
|
||||
|
||||
def _get_list_limit(self):
|
||||
return CONF.assignment.list_limit or CONF.list_limit
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_project_by_name(self, tenant_name, domain_id):
|
||||
"""Get a tenant by name.
|
||||
|
|
|
@ -88,7 +88,7 @@ class Catalog(kvs.Base, catalog.Driver):
|
|||
self.db.set('service_list', list(service_list))
|
||||
return service
|
||||
|
||||
def list_services(self):
|
||||
def list_services(self, hints):
|
||||
return [self.get_service(x) for x in self.db.get('service_list', [])]
|
||||
|
||||
def get_service(self, service_id):
|
||||
|
@ -119,7 +119,7 @@ class Catalog(kvs.Base, catalog.Driver):
|
|||
self.db.set('endpoint_list', list(endpoint_list))
|
||||
return endpoint
|
||||
|
||||
def list_endpoints(self):
|
||||
def list_endpoints(self, hints):
|
||||
return [self.get_endpoint(x) for x in self.db.get('endpoint_list', [])]
|
||||
|
||||
def get_endpoint(self, endpoint_id):
|
||||
|
|
|
@ -154,9 +154,11 @@ class Catalog(sql.Base, catalog.Driver):
|
|||
return ref.to_dict()
|
||||
|
||||
# Services
|
||||
def list_services(self):
|
||||
@sql.truncated
|
||||
def list_services(self, hints):
|
||||
session = db_session.get_session()
|
||||
services = session.query(Service).all()
|
||||
services = session.query(Service)
|
||||
services = self.filter_limit_query(Service, services, hints)
|
||||
return [s.to_dict() for s in list(services)]
|
||||
|
||||
def _get_service(self, session, service_id):
|
||||
|
@ -221,9 +223,11 @@ class Catalog(sql.Base, catalog.Driver):
|
|||
session = db_session.get_session()
|
||||
return self._get_endpoint(session, endpoint_id).to_dict()
|
||||
|
||||
def list_endpoints(self):
|
||||
@sql.truncated
|
||||
def list_endpoints(self, hints):
|
||||
session = db_session.get_session()
|
||||
endpoints = session.query(Endpoint)
|
||||
endpoints = self.filter_limit_query(Endpoint, endpoints, hints)
|
||||
return [e.to_dict() for e in list(endpoints)]
|
||||
|
||||
def update_endpoint(self, endpoint_id, endpoint_ref):
|
||||
|
|
|
@ -195,7 +195,7 @@ class ServiceV3(controller.V3Controller):
|
|||
@controller.filterprotected('type')
|
||||
def list_services(self, context, filters):
|
||||
hints = ServiceV3.build_driver_hints(context, filters)
|
||||
refs = self.catalog_api.list_services()
|
||||
refs = self.catalog_api.list_services(hints=hints)
|
||||
return ServiceV3.wrap_collection(context, refs, hints=hints)
|
||||
|
||||
@controller.protected()
|
||||
|
@ -248,7 +248,7 @@ class EndpointV3(controller.V3Controller):
|
|||
@controller.filterprotected('interface', 'service_id')
|
||||
def list_endpoints(self, context, filters):
|
||||
hints = EndpointV3.build_driver_hints(context, filters)
|
||||
refs = self.catalog_api.list_endpoints()
|
||||
refs = self.catalog_api.list_endpoints(hints=hints)
|
||||
return EndpointV3.wrap_collection(context, refs, hints=hints)
|
||||
|
||||
@controller.protected()
|
||||
|
|
|
@ -22,6 +22,7 @@ import abc
|
|||
import six
|
||||
|
||||
from keystone.common import dependency
|
||||
from keystone.common import driver_hints
|
||||
from keystone.common import manager
|
||||
from keystone import config
|
||||
from keystone import exception
|
||||
|
@ -99,6 +100,10 @@ class Manager(manager.Manager):
|
|||
except exception.NotFound:
|
||||
raise exception.ServiceNotFound(service_id=service_id)
|
||||
|
||||
@manager.response_truncated
|
||||
def list_services(self, hints=None):
|
||||
return self.driver.list_services(hints or driver_hints.Hints())
|
||||
|
||||
def create_endpoint(self, endpoint_id, endpoint_ref):
|
||||
try:
|
||||
return self.driver.create_endpoint(endpoint_id, endpoint_ref)
|
||||
|
@ -118,6 +123,10 @@ class Manager(manager.Manager):
|
|||
except exception.NotFound:
|
||||
raise exception.EndpointNotFound(endpoint_id=endpoint_id)
|
||||
|
||||
@manager.response_truncated
|
||||
def list_endpoints(self, hints=None):
|
||||
return self.driver.list_endpoints(hints or driver_hints.Hints())
|
||||
|
||||
def get_catalog(self, user_id, tenant_id, metadata=None):
|
||||
try:
|
||||
return self.driver.get_catalog(user_id, tenant_id, metadata)
|
||||
|
@ -129,6 +138,9 @@ class Manager(manager.Manager):
|
|||
class Driver(object):
|
||||
"""Interface description for an Catalog driver."""
|
||||
|
||||
def _get_list_limit(self):
|
||||
return CONF.catalog.list_limit or CONF.list_limit
|
||||
|
||||
@abc.abstractmethod
|
||||
def create_region(self, region_id, region_ref):
|
||||
"""Creates a new region.
|
||||
|
|
|
@ -57,7 +57,8 @@ FILE_OPTIONS = {
|
|||
default=600,
|
||||
help=("Sets the value of TCP_KEEPIDLE in seconds for each "
|
||||
"server socket. Only applies if tcp_keepalive is "
|
||||
"True. Not supported on OS X."))],
|
||||
"True. Not supported on OS X.")),
|
||||
cfg.IntOpt('list_limit', default=None)],
|
||||
'identity': [
|
||||
cfg.StrOpt('default_domain_id', default='default'),
|
||||
cfg.BoolOpt('domain_specific_drivers_enabled',
|
||||
|
@ -67,7 +68,8 @@ FILE_OPTIONS = {
|
|||
cfg.StrOpt('driver',
|
||||
default=('keystone.identity.backends'
|
||||
'.sql.Identity')),
|
||||
cfg.IntOpt('max_password_length', default=4096)],
|
||||
cfg.IntOpt('max_password_length', default=4096),
|
||||
cfg.IntOpt('list_limit', default=None)],
|
||||
'trust': [
|
||||
cfg.BoolOpt('enabled', default=True),
|
||||
cfg.StrOpt('driver',
|
||||
|
@ -139,7 +141,8 @@ FILE_OPTIONS = {
|
|||
# the backend
|
||||
cfg.StrOpt('driver', default=None),
|
||||
cfg.BoolOpt('caching', default=True),
|
||||
cfg.IntOpt('cache_time', default=None)],
|
||||
cfg.IntOpt('cache_time', default=None),
|
||||
cfg.IntOpt('list_limit', default=None)],
|
||||
'credential': [
|
||||
cfg.StrOpt('driver',
|
||||
default=('keystone.credential.backends'
|
||||
|
@ -157,7 +160,8 @@ FILE_OPTIONS = {
|
|||
|
||||
'policy': [
|
||||
cfg.StrOpt('driver',
|
||||
default='keystone.policy.backends.sql.Policy')],
|
||||
default='keystone.policy.backends.sql.Policy'),
|
||||
cfg.IntOpt('list_limit', default=None)],
|
||||
'ec2': [
|
||||
cfg.StrOpt('driver',
|
||||
default='keystone.contrib.ec2.backends.kvs.Ec2')],
|
||||
|
@ -274,7 +278,8 @@ FILE_OPTIONS = {
|
|||
cfg.StrOpt('template_file',
|
||||
default='default_catalog.templates'),
|
||||
cfg.StrOpt('driver',
|
||||
default='keystone.catalog.backends.sql.Catalog')],
|
||||
default='keystone.catalog.backends.sql.Catalog'),
|
||||
cfg.IntOpt('list_limit', default=None)],
|
||||
'kvs': [
|
||||
cfg.ListOpt('backends', default=[]),
|
||||
cfg.StrOpt('config_prefix', default='keystone.kvs'),
|
||||
|
|
|
@ -268,16 +268,16 @@ class V3Controller(wsgi.Application):
|
|||
|
||||
Returns the wrapped collection, which includes:
|
||||
- Executing any filtering not already carried out
|
||||
- Paginating if necessary
|
||||
- Truncate to a set limit if necessary
|
||||
- Adds 'self' links in every member
|
||||
- Adds 'next', 'self' and 'prev' links for the whole collection.
|
||||
|
||||
:param context: the current context, containing the original url path
|
||||
and query string
|
||||
:param refs: the list of members of the collection
|
||||
:param hints: list hints, containing any relevant
|
||||
filters. Any filters already satisfied by drivers
|
||||
will have been removed
|
||||
:param hints: list hints, containing any relevant filters and limit.
|
||||
Any filters already satisfied by managers will have been
|
||||
removed
|
||||
"""
|
||||
# Check if there are any filters in hints that were not
|
||||
# handled by the drivers. The driver will not have paginated or
|
||||
|
@ -287,7 +287,7 @@ class V3Controller(wsgi.Application):
|
|||
if hints is not None:
|
||||
refs = cls.filter_by_attributes(refs, hints)
|
||||
|
||||
refs = cls.paginate(context, refs)
|
||||
list_limited, refs = cls.limit(refs, hints)
|
||||
|
||||
for ref in refs:
|
||||
cls.wrap_member(context, ref)
|
||||
|
@ -297,17 +297,45 @@ class V3Controller(wsgi.Application):
|
|||
'next': None,
|
||||
'self': cls.base_url(path=context['path']),
|
||||
'previous': None}
|
||||
|
||||
if list_limited:
|
||||
container['truncated'] = True
|
||||
|
||||
return container
|
||||
|
||||
@classmethod
|
||||
def paginate(cls, context, refs):
|
||||
"""Paginates a list of references by page & per_page query strings."""
|
||||
# FIXME(dolph): client needs to support pagination first
|
||||
return refs
|
||||
def limit(cls, refs, hints):
|
||||
"""Limits a list of entities.
|
||||
|
||||
page = context['query_string'].get('page', 1)
|
||||
per_page = context['query_string'].get('per_page', 30)
|
||||
return refs[per_page * (page - 1):per_page * page]
|
||||
The underlying driver layer may have already truncated the collection
|
||||
for us, but in case it was unable to handle truncation we check here.
|
||||
|
||||
:param refs: the list of members of the collection
|
||||
:param hints: hints, containing, among other things, the limit
|
||||
requested
|
||||
|
||||
:returns: boolean indicating whether the list was truncated, as well
|
||||
as the list of (truncated if necessary) entities.
|
||||
|
||||
"""
|
||||
NOT_LIMITED = False
|
||||
LIMITED = True
|
||||
|
||||
if hints is None or hints.get_limit() is None:
|
||||
# No truncation was requested
|
||||
return NOT_LIMITED, refs
|
||||
|
||||
list_limit = hints.get_limit()
|
||||
if list_limit.get('truncated', False):
|
||||
# The driver did truncate the list
|
||||
return LIMITED, refs
|
||||
|
||||
if len(refs) > list_limit['limit']:
|
||||
# The driver layer wasn't able to truncate it for us, so we must
|
||||
# do it here
|
||||
return LIMITED, refs[:list_limit['limit']]
|
||||
|
||||
return NOT_LIMITED, refs
|
||||
|
||||
@classmethod
|
||||
def filter_by_attributes(cls, refs, hints):
|
||||
|
|
|
@ -21,14 +21,15 @@ class Hints(list):
|
|||
|
||||
Hints are modifiers that affect the return of entities from a
|
||||
list_<entities> operation. They are typically passed to a driver to give
|
||||
direction as to what filtering and pagination actions are being requested.
|
||||
direction as to what filtering, pagination or list limiting actions are
|
||||
being requested.
|
||||
|
||||
It is optional for a driver to action some or all of the list hints,
|
||||
but any filters that it does satisfy must be marked as such by calling
|
||||
removing the filter from the list.
|
||||
|
||||
A Hint object is a list of dicts, initially all of type 'filter', although
|
||||
other types may be added in the future. The list can be enumerated
|
||||
A Hint object is a list of dicts, initially of type 'filter' or 'limit',
|
||||
although other types may be added in the future. The list can be enumerated
|
||||
directly, or by using the filters() method which will guarantee to only
|
||||
return filters.
|
||||
|
||||
|
@ -60,3 +61,22 @@ class Hints(list):
|
|||
if (entry['type'] == 'filter' and entry['name'] == name and
|
||||
entry['comparator'] == 'equals'):
|
||||
return entry
|
||||
|
||||
def set_limit(self, limit, truncated=False):
|
||||
"""Set a limit to indicate the list should be truncated."""
|
||||
# We only allow one limit entry in the list, so if it already exists
|
||||
# we overwrite the old one
|
||||
for x in self:
|
||||
if x['type'] == 'limit':
|
||||
x['limit'] = limit
|
||||
x['truncated'] = truncated
|
||||
break
|
||||
else:
|
||||
self.append({'limit': limit, 'type': 'limit',
|
||||
'truncated': truncated})
|
||||
|
||||
def get_limit(self):
|
||||
"""Get the limit to which the list should be truncated."""
|
||||
for x in self:
|
||||
if x['type'] == 'limit':
|
||||
return x
|
||||
|
|
|
@ -19,6 +19,42 @@ import functools
|
|||
from keystone.openstack.common import importutils
|
||||
|
||||
|
||||
def response_truncated(f):
|
||||
"""Truncate the list returned by the wrapped function.
|
||||
|
||||
This is designed to wrap Manager list_{entity} methods to ensure that
|
||||
any list limits that are defined are passed to the driver layer. If a
|
||||
hints list is provided, the wrapper will insert the relevant limit into
|
||||
the hints so that the underlying driver call can try and honor it. If the
|
||||
driver does truncate the response, it will update the 'truncated' attribute
|
||||
in the 'limit' entry in the hints list, which enables the caller of this
|
||||
function to know if truncation has taken place. If, however, the driver
|
||||
layer is unable to perform truncation, the 'limit' entry is simply left in
|
||||
the hints list for the caller to handle.
|
||||
|
||||
A _get_list_limit() method is required to be present in the object class
|
||||
hierarchy, which returns the limit for this backend to which we will
|
||||
truncate.
|
||||
|
||||
If a hints list is not provided in the arguments of the wrapped call then
|
||||
any limits set in the config file are ignored. This allows internal use
|
||||
of such wrapped methods where the entire data set is needed as input for
|
||||
the calculations of some other API (e.g. get role assignments for a given
|
||||
project).
|
||||
|
||||
"""
|
||||
@functools.wraps(f)
|
||||
def wrapper(self, *args, **kwargs):
|
||||
if kwargs.get('hints') is None:
|
||||
return f(self, *args, **kwargs)
|
||||
|
||||
list_limit = self.driver._get_list_limit()
|
||||
if list_limit:
|
||||
kwargs['hints'].set_limit(list_limit)
|
||||
return f(self, *args, **kwargs)
|
||||
return wrapper
|
||||
|
||||
|
||||
class Manager(object):
|
||||
"""Base class for intermediary request layer.
|
||||
|
||||
|
|
|
@ -111,7 +111,7 @@ class XmlDeserializer(object):
|
|||
values = {}
|
||||
for k, v in six.iteritems(element.attrib):
|
||||
# boolean-looking attributes become booleans in JSON
|
||||
if k in ['enabled']:
|
||||
if k in ['enabled', 'truncated']:
|
||||
if v in ['true']:
|
||||
v = True
|
||||
elif v in ['false']:
|
||||
|
@ -143,6 +143,7 @@ class XmlDeserializer(object):
|
|||
return {'links': self._deserialize_links(element)}
|
||||
|
||||
links = None
|
||||
truncated = False
|
||||
for child in [self.walk_element(x) for x in element
|
||||
if not isinstance(x, ENTITY_TYPE)]:
|
||||
if list_item_tag:
|
||||
|
@ -152,7 +153,10 @@ class XmlDeserializer(object):
|
|||
if list_item_tag in child:
|
||||
values.append(child[list_item_tag])
|
||||
else:
|
||||
links = child['links']
|
||||
if 'links' in child:
|
||||
links = child['links']
|
||||
else:
|
||||
truncated = child['truncated']
|
||||
else:
|
||||
values = dict(values.items() + child.items())
|
||||
|
||||
|
@ -167,6 +171,9 @@ class XmlDeserializer(object):
|
|||
d['links'].setdefault('next')
|
||||
d['links'].setdefault('previous')
|
||||
|
||||
if truncated:
|
||||
d['truncated'] = truncated['truncated']
|
||||
|
||||
return d
|
||||
|
||||
|
||||
|
@ -178,17 +185,23 @@ class XmlSerializer(object):
|
|||
|
||||
"""
|
||||
links = None
|
||||
truncated = False
|
||||
# FIXME(dolph): skipping links for now
|
||||
for key in d.keys():
|
||||
if '_links' in key:
|
||||
d.pop(key)
|
||||
# FIXME(gyee): special-case links in collections
|
||||
# NOTE(gyee, henry-nash): special-case links and truncation
|
||||
# attribute in collections
|
||||
if 'links' == key:
|
||||
if links:
|
||||
# we have multiple links
|
||||
raise Exception('Multiple links found')
|
||||
links = d.pop(key)
|
||||
|
||||
if 'truncated' == key:
|
||||
if truncated:
|
||||
# we have multiple attributes
|
||||
raise Exception(_('Multiple truncation attributes found'))
|
||||
truncated = d.pop(key)
|
||||
assert len(d.keys()) == 1, ('Cannot encode more than one root '
|
||||
'element: %s' % d.keys())
|
||||
|
||||
|
@ -206,9 +219,11 @@ class XmlSerializer(object):
|
|||
|
||||
self.populate_element(root, d[name])
|
||||
|
||||
# FIXME(gyee): special-case links for now
|
||||
# NOTE(gyee, henry-nash): special-case links and truncation attribute
|
||||
if links:
|
||||
self._populate_links(root, links)
|
||||
if truncated:
|
||||
self._populate_truncated(root, truncated)
|
||||
|
||||
# TODO(dolph): you can get a doctype from lxml, using ElementTrees
|
||||
return '%s\n%s' % (DOCTYPE, etree.tostring(root, pretty_print=True))
|
||||
|
@ -223,6 +238,11 @@ class XmlSerializer(object):
|
|||
links.append(link)
|
||||
element.append(links)
|
||||
|
||||
def _populate_truncated(self, element, truncated_value):
|
||||
truncated = etree.Element('truncated')
|
||||
self._populate_bool(truncated, 'truncated', truncated_value)
|
||||
element.append(truncated)
|
||||
|
||||
def _populate_list(self, element, k, v):
|
||||
"""Populates an element with a key & list value."""
|
||||
# spec has a lot of inconsistency here!
|
||||
|
|
|
@ -154,6 +154,47 @@ def transaction(expire_on_commit=False):
|
|||
yield session
|
||||
|
||||
|
||||
def truncated(f):
|
||||
"""Ensure list truncation is detected in Driver list entity methods.
|
||||
|
||||
This is designed to wrap and sql Driver list_{entity} methods in order to
|
||||
calculate if the resultant list has been truncated. Provided a limit dict
|
||||
is found in the hints list, we increment the limit by one so as to ask the
|
||||
wrapped function for one more entity than the limit, and then once the list
|
||||
has been generated, we check to see if the original limit has been
|
||||
exceeded, in which case we truncate back to that limit and set the
|
||||
'truncated' boolean to 'true' in the hints limit dict.
|
||||
|
||||
"""
|
||||
@functools.wraps(f)
|
||||
def wrapper(self, *args, **kwargs):
|
||||
if not hasattr(args[0], 'get_limit'):
|
||||
raise exception.UnexpectedError(
|
||||
_('Cannot truncate a driver call without hints list as '
|
||||
'first parameter after self '))
|
||||
|
||||
hints = args[0]
|
||||
limit_dict = hints.get_limit()
|
||||
if limit_dict is None:
|
||||
return f(self, *args, **kwargs)
|
||||
|
||||
# A limit is set, so ask for one more entry than we need
|
||||
list_limit = limit_dict['limit']
|
||||
hints.set_limit(list_limit + 1)
|
||||
ref_list = f(self, *args, **kwargs)
|
||||
|
||||
# If we got more than the original limit then trim back the list and
|
||||
# mark it truncated. In both cases, make sure we set the limit back
|
||||
# to its original value.
|
||||
if len(ref_list) > list_limit:
|
||||
hints.set_limit(list_limit, truncated=True)
|
||||
return ref_list[:list_limit]
|
||||
else:
|
||||
hints.set_limit(list_limit)
|
||||
return ref_list
|
||||
return wrapper
|
||||
|
||||
|
||||
# Backends
|
||||
class Base(object):
|
||||
def _filter(self, model, query, hints):
|
||||
|
@ -245,23 +286,56 @@ class Base(object):
|
|||
|
||||
return query
|
||||
|
||||
def filter_query(self, model, query, hints):
|
||||
"""Applies filtering to a query.
|
||||
def _limit(self, query, hints):
|
||||
"""Applies a limit to a query.
|
||||
|
||||
:param model: table model
|
||||
:param query: query to apply filters to
|
||||
:param hints: contains the list of filters yet to be satisfied.
|
||||
Any filters satisfied here will be removed so that
|
||||
the caller will know if any filters remain.
|
||||
:param hints: contains the list of filters and limit details.
|
||||
|
||||
:returns updated query
|
||||
|
||||
"""
|
||||
if hints is not None:
|
||||
query = self._filter(model, query, hints)
|
||||
# NOTE(henry-nash): If we were to implement pagination, then we
|
||||
# we would expand this method to support pagination and limiting.
|
||||
|
||||
# If we satisfied all the filters, set an upper limit if supplied
|
||||
list_limit = hints.get_limit()
|
||||
if list_limit:
|
||||
query = query.limit(list_limit['limit'])
|
||||
return query
|
||||
|
||||
def filter_limit_query(self, model, query, hints):
|
||||
"""Applies filtering and limit to a query.
|
||||
|
||||
:param model: table model
|
||||
:param query: query to apply filters to
|
||||
:param hints: contains the list of filters and limit details. This may
|
||||
be None, indicating that there are no filters or limits
|
||||
to be applied. If it's not None, then any filters
|
||||
satisfied here will be removed so that the caller will
|
||||
know if any filters remain.
|
||||
|
||||
:returns: updated query
|
||||
|
||||
"""
|
||||
if hints is None:
|
||||
return query
|
||||
|
||||
# First try and satisfy any filters
|
||||
query = self._filter(model, query, hints)
|
||||
|
||||
# NOTE(henry-nash): Any unsatisfied filters will have been left in
|
||||
# the hints list for the controller to handle. We can only try and
|
||||
# limit here if all the filters are already satisfied since, if not,
|
||||
# doing so might mess up the final results. If there are still
|
||||
# unsatisfied filters, we have to leave any limiting to the controller
|
||||
# as well.
|
||||
|
||||
if not hints.filters():
|
||||
return self._limit(query, hints)
|
||||
else:
|
||||
return query
|
||||
|
||||
|
||||
def handle_conflicts(conflict_type='object'):
|
||||
"""Converts select sqlalchemy exceptions into HTTP 409 Conflict."""
|
||||
|
|
|
@ -19,6 +19,7 @@ import json
|
|||
|
||||
from keystone.common import controller
|
||||
from keystone.common import dependency
|
||||
from keystone.common import driver_hints
|
||||
from keystone import exception
|
||||
|
||||
|
||||
|
@ -80,9 +81,13 @@ class CredentialV3(controller.V3Controller):
|
|||
|
||||
@controller.protected()
|
||||
def list_credentials(self, context):
|
||||
# NOTE(henry-nash): Since there are no filters for credentials, we
|
||||
# shouldn't limit the output, hence we don't pass a hints list into
|
||||
# the driver.
|
||||
refs = self.credential_api.list_credentials()
|
||||
ret_refs = [self._blob_to_json(r) for r in refs]
|
||||
return CredentialV3.wrap_collection(context, ret_refs)
|
||||
return CredentialV3.wrap_collection(context, ret_refs,
|
||||
driver_hints.Hints())
|
||||
|
||||
@controller.protected()
|
||||
def get_credential(self, context, credential_id):
|
||||
|
|
|
@ -124,10 +124,11 @@ class Identity(sql.Base, identity.Driver):
|
|||
session.add(user_ref)
|
||||
return identity.filter_user(user_ref.to_dict())
|
||||
|
||||
@sql.truncated
|
||||
def list_users(self, hints):
|
||||
session = db_session.get_session()
|
||||
query = session.query(User)
|
||||
user_refs = self.filter_query(User, query, hints)
|
||||
user_refs = self.filter_limit_query(User, query, hints)
|
||||
return [identity.filter_user(x.to_dict()) for x in user_refs]
|
||||
|
||||
def _get_user(self, session, user_id):
|
||||
|
@ -256,10 +257,11 @@ class Identity(sql.Base, identity.Driver):
|
|||
session.add(ref)
|
||||
return ref.to_dict()
|
||||
|
||||
@sql.truncated
|
||||
def list_groups(self, hints):
|
||||
session = db_session.get_session()
|
||||
query = session.query(Group)
|
||||
refs = self.filter_query(Group, query, hints)
|
||||
refs = self.filter_limit_query(Group, query, hints)
|
||||
return [ref.to_dict() for ref in refs]
|
||||
|
||||
def _get_group(self, session, group_id):
|
||||
|
|
|
@ -363,6 +363,7 @@ class Manager(manager.Manager):
|
|||
ref = self._set_domain_id(ref, domain_id)
|
||||
return ref
|
||||
|
||||
@manager.response_truncated
|
||||
@domains_configured
|
||||
def list_users(self, domain_scope=None, hints=None):
|
||||
domain_id, driver = self._get_domain_id_and_driver(domain_scope)
|
||||
|
@ -463,6 +464,7 @@ class Manager(manager.Manager):
|
|||
driver.remove_user_from_group(user_id, group_id)
|
||||
self.token_api.delete_tokens_for_user(user_id)
|
||||
|
||||
@manager.response_truncated
|
||||
@domains_configured
|
||||
def list_groups_for_user(self, user_id, domain_scope=None,
|
||||
hints=None):
|
||||
|
@ -477,6 +479,7 @@ class Manager(manager.Manager):
|
|||
ref_list = self._set_domain_id(ref_list, domain_id)
|
||||
return ref_list
|
||||
|
||||
@manager.response_truncated
|
||||
@domains_configured
|
||||
def list_groups(self, domain_scope=None, hints=None):
|
||||
domain_id, driver = self._get_domain_id_and_driver(domain_scope)
|
||||
|
@ -489,6 +492,7 @@ class Manager(manager.Manager):
|
|||
ref_list = self._set_domain_id(ref_list, domain_id)
|
||||
return ref_list
|
||||
|
||||
@manager.response_truncated
|
||||
@domains_configured
|
||||
def list_users_in_group(self, group_id, domain_scope=None,
|
||||
hints=None):
|
||||
|
@ -638,6 +642,9 @@ class Manager(manager.Manager):
|
|||
class Driver(object):
|
||||
"""Interface description for an Identity driver."""
|
||||
|
||||
def _get_list_limit(self):
|
||||
return CONF.identity.list_limit or CONF.list_limit
|
||||
|
||||
@abc.abstractmethod
|
||||
def authenticate(self, user_id, password):
|
||||
"""Authenticate a given user and password.
|
||||
|
|
|
@ -35,10 +35,7 @@ class PolicyV3(controller.V3Controller):
|
|||
@controller.filterprotected('type')
|
||||
def list_policies(self, context, filters):
|
||||
hints = PolicyV3.build_driver_hints(context, filters)
|
||||
# We don't bother passing the hints in, since this would be
|
||||
# a highly unlikely filter to use - wrap_collection() can
|
||||
# handle if required.
|
||||
refs = self.policy_api.list_policies()
|
||||
refs = self.policy_api.list_policies(hints=hints)
|
||||
return PolicyV3.wrap_collection(context, refs, hints=hints)
|
||||
|
||||
@controller.protected()
|
||||
|
|
|
@ -55,6 +55,13 @@ class Manager(manager.Manager):
|
|||
except exception.NotFound:
|
||||
raise exception.PolicyNotFound(policy_id=policy_id)
|
||||
|
||||
@manager.response_truncated
|
||||
def list_policies(self, hints=None):
|
||||
# NOTE(henry-nash): Since the advantage of filtering or list limiting
|
||||
# of policies at the driver level is minimal, we leave this to the
|
||||
# caller.
|
||||
return self.driver.list_policies()
|
||||
|
||||
def delete_policy(self, policy_id):
|
||||
try:
|
||||
return self.driver.delete_policy(policy_id)
|
||||
|
@ -65,6 +72,9 @@ class Manager(manager.Manager):
|
|||
@six.add_metaclass(abc.ABCMeta)
|
||||
class Driver(object):
|
||||
|
||||
def _get_list_limit(self):
|
||||
return CONF.policy.list_limit or CONF.list_limit
|
||||
|
||||
@abc.abstractmethod
|
||||
def enforce(self, context, credentials, action, target):
|
||||
"""Verify that a user is authorized to perform action.
|
||||
|
|
|
@ -4093,3 +4093,76 @@ class FilterTests(filtering.FilterTests):
|
|||
|
||||
groups = self.identity_api.list_groups()
|
||||
self.assertTrue(len(groups) > 0)
|
||||
|
||||
|
||||
class LimitTests(filtering.FilterTests):
|
||||
def setUp(self):
|
||||
"""Setup for Limit Test Cases."""
|
||||
|
||||
self.domain1 = {'id': uuid.uuid4().hex, 'name': uuid.uuid4().hex}
|
||||
self.assignment_api.create_domain(self.domain1['id'], self.domain1)
|
||||
self.addCleanup(self.clean_up_domain)
|
||||
|
||||
self.entity_lists = {}
|
||||
self.domain1_entity_lists = {}
|
||||
|
||||
for entity in ['user', 'group', 'project']:
|
||||
# Create 20 entities, 14 of which are in domain1
|
||||
self.entity_lists[entity] = self._create_test_data(entity, 6)
|
||||
self.domain1_entity_lists[entity] = self._create_test_data(
|
||||
entity, 14, self.domain1['id'])
|
||||
# Make sure we clean up when finished
|
||||
self.addCleanup(self.clean_up_entity, entity)
|
||||
|
||||
def clean_up_domain(self):
|
||||
"""Clean up domain test data from Limit Test Cases."""
|
||||
|
||||
self.domain1['enabled'] = False
|
||||
self.assignment_api.update_domain(self.domain1['id'], self.domain1)
|
||||
self.assignment_api.delete_domain(self.domain1['id'])
|
||||
|
||||
def clean_up_entity(self, entity):
|
||||
"""Clean up entity test data from Limit Test Cases."""
|
||||
|
||||
self._delete_test_data(entity, self.entity_lists[entity])
|
||||
self._delete_test_data(entity, self.domain1_entity_lists[entity])
|
||||
|
||||
def _test_list_entity_filtered_and_limited(self, entity):
|
||||
self.opt(list_limit=10)
|
||||
# Should get back just 10 entities in domain1
|
||||
hints = driver_hints.Hints()
|
||||
hints.add_filter('domain_id', self.domain1['id'])
|
||||
entities = self._list_entities(entity)(hints=hints)
|
||||
self.assertEqual(len(entities), hints.get_limit()['limit'])
|
||||
self.assertTrue(hints.get_limit()['truncated'])
|
||||
self._match_with_list(entities, self.domain1_entity_lists[entity])
|
||||
|
||||
# Override with driver specific limit
|
||||
if entity == 'project':
|
||||
self.opt_in_group('assignment', list_limit=5)
|
||||
else:
|
||||
self.opt_in_group('identity', list_limit=5)
|
||||
|
||||
# Should get back just 5 users in domain1
|
||||
hints = driver_hints.Hints()
|
||||
hints.add_filter('domain_id', self.domain1['id'])
|
||||
entities = self._list_entities(entity)(hints=hints)
|
||||
self.assertEqual(len(entities), hints.get_limit()['limit'])
|
||||
self._match_with_list(entities, self.domain1_entity_lists[entity])
|
||||
|
||||
# Finally, let's pretend we want to get the full list of entities,
|
||||
# even with the limits set, as part of some internal calculation.
|
||||
# Calling the API without a hints list should achieve this, and
|
||||
# return at least the 20 entries we created (there may be other
|
||||
# entities lying around created by other tests/setup).
|
||||
entities = self._list_entities(entity)()
|
||||
self.assertTrue(len(entities) >= 20)
|
||||
|
||||
def test_list_users_filtered_and_limited(self):
|
||||
self._test_list_entity_filtered_and_limited('user')
|
||||
|
||||
def test_list_groups_filtered_and_limited(self):
|
||||
self._test_list_entity_filtered_and_limited('group')
|
||||
|
||||
def test_list_projects_filtered_and_limited(self):
|
||||
self._test_list_entity_filtered_and_limited('project')
|
||||
|
|
|
@ -486,3 +486,9 @@ class SqlTokenCacheInvalidation(SqlTests, test_backend.TokenCacheInvalidation):
|
|||
|
||||
class SqlFilterTests(SqlTests, test_backend.FilterTests):
|
||||
pass
|
||||
|
||||
|
||||
class SqlLimitTests(SqlTests, test_backend.LimitTests):
|
||||
def setUp(self):
|
||||
super(SqlLimitTests, self).setUp()
|
||||
test_backend.LimitTests.setUp(self)
|
||||
|
|
|
@ -49,3 +49,16 @@ class ListHintsTests(test.TestCase):
|
|||
hints2.add_filter('t4', 'data1')
|
||||
hints2.add_filter('t5', 'data2')
|
||||
self.assertEqual(len(hints.filters()), 2)
|
||||
|
||||
def test_limits(self):
|
||||
hints = driver_hints.Hints()
|
||||
self.assertIsNone(hints.get_limit())
|
||||
hints.set_limit(10)
|
||||
self.assertEqual(hints.get_limit()['limit'], 10)
|
||||
self.assertFalse(hints.get_limit()['truncated'])
|
||||
hints.set_limit(11)
|
||||
self.assertEqual(hints.get_limit()['limit'], 11)
|
||||
self.assertFalse(hints.get_limit()['truncated'])
|
||||
hints.set_limit(10, truncated=True)
|
||||
self.assertEqual(hints.get_limit()['limit'], 10)
|
||||
self.assertTrue(hints.get_limit()['truncated'])
|
||||
|
|
|
@ -31,6 +31,7 @@ CONF = config.CONF
|
|||
class IdentityTestFilteredCase(filtering.FilterTests,
|
||||
test_v3.RestfulTestCase):
|
||||
"""Test filter enforcement on the v3 Identity API."""
|
||||
content_type = 'json'
|
||||
|
||||
def setUp(self):
|
||||
"""Setup for Identity Filter Test Cases."""
|
||||
|
@ -299,3 +300,141 @@ class IdentityTestFilteredCase(filtering.FilterTests,
|
|||
url_by_name = "/groups"
|
||||
r = self.get(url_by_name, auth=self.auth)
|
||||
self.assertTrue(len(r.result.get('groups')) > 0)
|
||||
|
||||
|
||||
class IdentityTestListLimitCase(IdentityTestFilteredCase):
|
||||
"""Test list limiting enforcement on the v3 Identity API."""
|
||||
content_type = 'json'
|
||||
|
||||
def setUp(self):
|
||||
"""Setup for Identity Limit Test Cases."""
|
||||
|
||||
super(IdentityTestListLimitCase, self).setUp()
|
||||
|
||||
self._set_policy({"identity:list_users": [],
|
||||
"identity:list_groups": [],
|
||||
"identity:list_projects": [],
|
||||
"identity:list_services": [],
|
||||
"identity:list_policies": []})
|
||||
|
||||
# Create 10 entries for each of the entities we are going to test
|
||||
self.ENTITY_TYPES = ['user', 'group', 'project']
|
||||
self.entity_lists = {}
|
||||
for entity in self.ENTITY_TYPES:
|
||||
self.entity_lists[entity] = self._create_test_data(entity, 10)
|
||||
# Make sure we clean up when finished
|
||||
self.addCleanup(self.clean_up_entity, entity)
|
||||
|
||||
self.service_list = []
|
||||
self.addCleanup(self.clean_up_service)
|
||||
for _ in range(10):
|
||||
new_entity = {'id': uuid.uuid4().hex, 'type': uuid.uuid4().hex}
|
||||
service = self.catalog_api.create_service(new_entity['id'],
|
||||
new_entity)
|
||||
self.service_list.append(service)
|
||||
|
||||
self.policy_list = []
|
||||
self.addCleanup(self.clean_up_policy)
|
||||
for _ in range(10):
|
||||
new_entity = {'id': uuid.uuid4().hex, 'type': uuid.uuid4().hex,
|
||||
'blob': uuid.uuid4().hex}
|
||||
policy = self.policy_api.create_policy(new_entity['id'],
|
||||
new_entity)
|
||||
self.policy_list.append(policy)
|
||||
|
||||
def clean_up_entity(self, entity):
|
||||
"""Clean up entity test data from Identity Limit Test Cases."""
|
||||
|
||||
self._delete_test_data(entity, self.entity_lists[entity])
|
||||
|
||||
def clean_up_service(self):
|
||||
"""Clean up service test data from Identity Limit Test Cases."""
|
||||
|
||||
for service in self.service_list:
|
||||
self.catalog_api.delete_service(service['id'])
|
||||
|
||||
def clean_up_policy(self):
|
||||
"""Clean up policy test data from Identity Limit Test Cases."""
|
||||
|
||||
for policy in self.policy_list:
|
||||
self.policy_api.delete_policy(policy['id'])
|
||||
|
||||
def _test_entity_list_limit(self, entity, driver):
|
||||
"""GET /<entities> (limited)
|
||||
|
||||
Test Plan:
|
||||
|
||||
- For the specified type of entity:
|
||||
- Update policy for no protection on api
|
||||
- Add a bunch of entities
|
||||
- Set the global list limit to 5, and check that getting all
|
||||
- entities only returns 5
|
||||
- Set the driver list_limit to 4, and check that now only 4 are
|
||||
- returned
|
||||
|
||||
"""
|
||||
if entity == 'policy':
|
||||
plural = 'policies'
|
||||
else:
|
||||
plural = '%ss' % entity
|
||||
|
||||
self.opt(list_limit=5)
|
||||
self.opt_in_group(driver, list_limit=None)
|
||||
r = self.get('/%s' % plural, auth=self.auth)
|
||||
self.assertEqual(len(r.result.get(plural)), 5)
|
||||
self.assertIs(r.result.get('truncated'), True)
|
||||
|
||||
self.opt_in_group(driver, list_limit=4)
|
||||
r = self.get('/%s' % plural, auth=self.auth)
|
||||
self.assertEqual(len(r.result.get(plural)), 4)
|
||||
self.assertIs(r.result.get('truncated'), True)
|
||||
|
||||
def test_users_list_limit(self):
|
||||
self._test_entity_list_limit('user', 'identity')
|
||||
|
||||
def test_groups_list_limit(self):
|
||||
self._test_entity_list_limit('group', 'identity')
|
||||
|
||||
def test_projects_list_limit(self):
|
||||
self._test_entity_list_limit('project', 'assignment')
|
||||
|
||||
def test_services_list_limit(self):
|
||||
self._test_entity_list_limit('service', 'catalog')
|
||||
|
||||
def test_non_driver_list_limit(self):
|
||||
"""Check list can be limited without driver level support.
|
||||
|
||||
Policy limiting is not done at the driver level (since it
|
||||
really isn't worth doing it there). So use this as a test
|
||||
for ensuring the controller level will successfully limit
|
||||
in this case.
|
||||
|
||||
"""
|
||||
self._test_entity_list_limit('policy', 'policy')
|
||||
|
||||
def test_no_limit(self):
|
||||
"""Check truncated attribute not set when list not limited."""
|
||||
|
||||
r = self.get('/services', auth=self.auth)
|
||||
self.assertEqual(len(r.result.get('services')), 10)
|
||||
self.assertIsNone(r.result.get('truncated'))
|
||||
|
||||
def test_at_limit(self):
|
||||
"""Check truncated attribute not set when list at max size."""
|
||||
|
||||
# Test this by overriding the general limit with a higher
|
||||
# driver-specific limit (allowing all entities to be returned
|
||||
# in the collection), which should result in a non truncated list
|
||||
self.opt(list_limit=5)
|
||||
self.opt_in_group('catalog', list_limit=10)
|
||||
r = self.get('/services', auth=self.auth)
|
||||
self.assertEqual(len(r.result.get('services')), 10)
|
||||
self.assertIsNone(r.result.get('truncated'))
|
||||
|
||||
|
||||
class IdentityTestFilteredCaseXML(IdentityTestFilteredCase):
|
||||
content_type = 'xml'
|
||||
|
||||
|
||||
class IdentityTestListLimitCaseXML(IdentityTestListLimitCase):
|
||||
content_type = 'xml'
|
||||
|
|
Loading…
Reference in New Issue