Implement list limiting support in driver backends

Provides an optional limit to the number of rows that will be
returned by a backend from a list_{entity} call.  The limit is specified
in the configuration file, and allows for an overall general limit as
well as an individual limit for a given driver.  By default, there is
no limit.

Limitations:
    - The list limit is not yet handled by LDAP, rather this remains
      implemented in the final wrap collection - a subsequent patch
      will provide the support in the LDAP drivers

Implements bp list-limiting

Change-Id: I7ca76a8da4260242e578c44103b26257f7e2a5d5
This commit is contained in:
Henry Nash 2013-08-11 10:26:31 +01:00
parent 1923a3f5ba
commit 0dec4c8be9
24 changed files with 563 additions and 50 deletions

View File

@ -694,6 +694,24 @@ should be set to one of the following modes:
mechanism called ``named``.
Limiting the number of entities returned in a collection
--------------------------------------------------------
Keystone provides a method of setting a limit to the number of entities
returned in a collection, which is useful to prevent overly long response
times for list queries that have not specified a sufficently narrow filter.
This limit can be set globally by setting ``list_limit`` in the default section
of ``keystone.conf``, with no limit set by default. Individual driver
sections may override this global value with a specific limit, for example::
[assignment]
list_limit = 100
If a response to ``list_{entity}`` call has been truncted, then the response
status code will still be 200 (OK), but the ``truncated`` attribute in the
collection will be set to ``true``.
Sample Configuration Files
--------------------------

View File

@ -144,6 +144,21 @@ The contract for a driver for ``list_{entity}`` methods is therefore:
list by filtering for one or more of the specified filters in the passed
Hints reference, and removing any such satisfied filters.
Entity list truncation by drivers
---------------------------------
Keystone supports the ability for a deployment to restrict the number of
entries returned from ``list_{entity}`` methods, typically to prevent poorly
formed searches (e.g. without sufficient filters) from becoming a performance
issue.
These limits are set in the configuration file, either for a specific driver or
across all drivers. These limits are read at the Manager level and passed into
individual drivers as part of the Hints list object. A driver should try and
honor any such limit if possible, but if it is unable to do so then it may
ignore it (and the truncation of the returned list of entities will happen at
the controller level).
Testing
-------

View File

@ -61,6 +61,12 @@
# If true, use synchronous mode for sqlite
# sqlite_synchronous = True
# The maximum number of entities that will be returned in a collection can be
# set with list_limit, with no limit set by default. This global limit may be
# then overridden for a specific driver, by specifying a list_limit in the
# appropriate section (e.g. [assignment])
# list_limit =
# === Logging Options ===
# Print debugging output
# (includes plaintext request logging, potentially including passwords)
@ -222,6 +228,9 @@
# Maximum supported length for user passwords; decrease to improve performance.
# max_password_length = 4096
# Maximum number of entities that will be returned in an identity collection
# list_limit =
[credential]
# driver = keystone.credential.backends.sql.Credential
@ -245,6 +254,9 @@
# template_file = default_catalog.templates
# Maximum number of entities that will be returned in a catalog collection
# list_limit =
[endpoint_filter]
# extension for creating associations between project and endpoints in order to
# provide a tailored catalog for project-scoped token requests.
@ -325,6 +337,9 @@
[policy]
# driver = keystone.policy.backends.sql.Policy
# Maximum number of entities that will be returned in a policy collection
# list_limit =
[ec2]
# driver = keystone.contrib.ec2.backends.kvs.Ec2
@ -338,6 +353,9 @@
# Assignment specific cache time-to-live (TTL) in seconds.
# cache_time =
# Maximum number of entities that will be returned in an assignment collection
# list_limit =
[oauth1]
# driver = keystone.contrib.oauth1.backends.sql.OAuth1

View File

@ -209,10 +209,11 @@ class Assignment(sql.Base, assignment.Driver):
self._update_metadata(session, user_id, project_id, metadata_ref,
domain_id, group_id)
@sql.truncated
def list_projects(self, hints):
with sql.transaction() as session:
query = session.query(Project)
project_refs = self.filter_query(Project, query, hints)
project_refs = self.filter_limit_query(Project, query, hints)
return [project_ref.to_dict() for project_ref in project_refs]
def list_projects_in_domain(self, domain_id):
@ -528,10 +529,11 @@ class Assignment(sql.Base, assignment.Driver):
session.add(ref)
return ref.to_dict()
@sql.truncated
def list_domains(self, hints):
with sql.transaction() as session:
query = session.query(Domain)
refs = self.filter_query(Domain, query, hints)
refs = self.filter_limit_query(Domain, query, hints)
return [ref.to_dict() for ref in refs]
def _get_domain(self, session, domain_id):
@ -581,10 +583,11 @@ class Assignment(sql.Base, assignment.Driver):
session.add(ref)
return ref.to_dict()
@sql.truncated
def list_roles(self, hints):
with sql.transaction() as session:
query = session.query(Role)
refs = self.filter_query(Role, query, hints)
refs = self.filter_limit_query(Role, query, hints)
return [ref.to_dict() for ref in refs]
def _get_role(self, session, role_id):

View File

@ -266,6 +266,8 @@ class Manager(manager.Manager):
"exist."),
role_id)
# TODO(henry-nash): We might want to consider list limiting this at some
# point in the future.
def list_projects_for_user(self, user_id, hints=None):
# NOTE(henry-nash): In order to get a complete list of user projects,
# the driver will need to look at group assignments. To avoid cross
@ -296,6 +298,7 @@ class Manager(manager.Manager):
self.get_domain_by_name.set(ret, self, ret['name'])
return ret
@manager.response_truncated
def list_domains(self, hints=None):
return self.driver.list_domains(hints or driver_hints.Hints())
@ -392,6 +395,7 @@ class Manager(manager.Manager):
{'userid': user['id'],
'domainid': domain_id})
@manager.response_truncated
def list_projects(self, hints=None):
return self.driver.list_projects(hints or driver_hints.Hints())
@ -427,6 +431,7 @@ class Manager(manager.Manager):
self.get_role.set(ret, self, role_id)
return ret
@manager.response_truncated
def list_roles(self, hints=None):
return self.driver.list_roles(hints or driver_hints.Hints())
@ -590,6 +595,9 @@ class Driver(object):
inherited).items()))
return [dict(r) for r in role_set]
def _get_list_limit(self):
return CONF.assignment.list_limit or CONF.list_limit
@abc.abstractmethod
def get_project_by_name(self, tenant_name, domain_id):
"""Get a tenant by name.

View File

@ -88,7 +88,7 @@ class Catalog(kvs.Base, catalog.Driver):
self.db.set('service_list', list(service_list))
return service
def list_services(self):
def list_services(self, hints):
return [self.get_service(x) for x in self.db.get('service_list', [])]
def get_service(self, service_id):
@ -119,7 +119,7 @@ class Catalog(kvs.Base, catalog.Driver):
self.db.set('endpoint_list', list(endpoint_list))
return endpoint
def list_endpoints(self):
def list_endpoints(self, hints):
return [self.get_endpoint(x) for x in self.db.get('endpoint_list', [])]
def get_endpoint(self, endpoint_id):

View File

@ -154,9 +154,11 @@ class Catalog(sql.Base, catalog.Driver):
return ref.to_dict()
# Services
def list_services(self):
@sql.truncated
def list_services(self, hints):
session = db_session.get_session()
services = session.query(Service).all()
services = session.query(Service)
services = self.filter_limit_query(Service, services, hints)
return [s.to_dict() for s in list(services)]
def _get_service(self, session, service_id):
@ -221,9 +223,11 @@ class Catalog(sql.Base, catalog.Driver):
session = db_session.get_session()
return self._get_endpoint(session, endpoint_id).to_dict()
def list_endpoints(self):
@sql.truncated
def list_endpoints(self, hints):
session = db_session.get_session()
endpoints = session.query(Endpoint)
endpoints = self.filter_limit_query(Endpoint, endpoints, hints)
return [e.to_dict() for e in list(endpoints)]
def update_endpoint(self, endpoint_id, endpoint_ref):

View File

@ -195,7 +195,7 @@ class ServiceV3(controller.V3Controller):
@controller.filterprotected('type')
def list_services(self, context, filters):
hints = ServiceV3.build_driver_hints(context, filters)
refs = self.catalog_api.list_services()
refs = self.catalog_api.list_services(hints=hints)
return ServiceV3.wrap_collection(context, refs, hints=hints)
@controller.protected()
@ -248,7 +248,7 @@ class EndpointV3(controller.V3Controller):
@controller.filterprotected('interface', 'service_id')
def list_endpoints(self, context, filters):
hints = EndpointV3.build_driver_hints(context, filters)
refs = self.catalog_api.list_endpoints()
refs = self.catalog_api.list_endpoints(hints=hints)
return EndpointV3.wrap_collection(context, refs, hints=hints)
@controller.protected()

View File

@ -22,6 +22,7 @@ import abc
import six
from keystone.common import dependency
from keystone.common import driver_hints
from keystone.common import manager
from keystone import config
from keystone import exception
@ -99,6 +100,10 @@ class Manager(manager.Manager):
except exception.NotFound:
raise exception.ServiceNotFound(service_id=service_id)
@manager.response_truncated
def list_services(self, hints=None):
return self.driver.list_services(hints or driver_hints.Hints())
def create_endpoint(self, endpoint_id, endpoint_ref):
try:
return self.driver.create_endpoint(endpoint_id, endpoint_ref)
@ -118,6 +123,10 @@ class Manager(manager.Manager):
except exception.NotFound:
raise exception.EndpointNotFound(endpoint_id=endpoint_id)
@manager.response_truncated
def list_endpoints(self, hints=None):
return self.driver.list_endpoints(hints or driver_hints.Hints())
def get_catalog(self, user_id, tenant_id, metadata=None):
try:
return self.driver.get_catalog(user_id, tenant_id, metadata)
@ -129,6 +138,9 @@ class Manager(manager.Manager):
class Driver(object):
"""Interface description for an Catalog driver."""
def _get_list_limit(self):
return CONF.catalog.list_limit or CONF.list_limit
@abc.abstractmethod
def create_region(self, region_id, region_ref):
"""Creates a new region.

View File

@ -57,7 +57,8 @@ FILE_OPTIONS = {
default=600,
help=("Sets the value of TCP_KEEPIDLE in seconds for each "
"server socket. Only applies if tcp_keepalive is "
"True. Not supported on OS X."))],
"True. Not supported on OS X.")),
cfg.IntOpt('list_limit', default=None)],
'identity': [
cfg.StrOpt('default_domain_id', default='default'),
cfg.BoolOpt('domain_specific_drivers_enabled',
@ -67,7 +68,8 @@ FILE_OPTIONS = {
cfg.StrOpt('driver',
default=('keystone.identity.backends'
'.sql.Identity')),
cfg.IntOpt('max_password_length', default=4096)],
cfg.IntOpt('max_password_length', default=4096),
cfg.IntOpt('list_limit', default=None)],
'trust': [
cfg.BoolOpt('enabled', default=True),
cfg.StrOpt('driver',
@ -139,7 +141,8 @@ FILE_OPTIONS = {
# the backend
cfg.StrOpt('driver', default=None),
cfg.BoolOpt('caching', default=True),
cfg.IntOpt('cache_time', default=None)],
cfg.IntOpt('cache_time', default=None),
cfg.IntOpt('list_limit', default=None)],
'credential': [
cfg.StrOpt('driver',
default=('keystone.credential.backends'
@ -157,7 +160,8 @@ FILE_OPTIONS = {
'policy': [
cfg.StrOpt('driver',
default='keystone.policy.backends.sql.Policy')],
default='keystone.policy.backends.sql.Policy'),
cfg.IntOpt('list_limit', default=None)],
'ec2': [
cfg.StrOpt('driver',
default='keystone.contrib.ec2.backends.kvs.Ec2')],
@ -274,7 +278,8 @@ FILE_OPTIONS = {
cfg.StrOpt('template_file',
default='default_catalog.templates'),
cfg.StrOpt('driver',
default='keystone.catalog.backends.sql.Catalog')],
default='keystone.catalog.backends.sql.Catalog'),
cfg.IntOpt('list_limit', default=None)],
'kvs': [
cfg.ListOpt('backends', default=[]),
cfg.StrOpt('config_prefix', default='keystone.kvs'),

View File

@ -268,16 +268,16 @@ class V3Controller(wsgi.Application):
Returns the wrapped collection, which includes:
- Executing any filtering not already carried out
- Paginating if necessary
- Truncate to a set limit if necessary
- Adds 'self' links in every member
- Adds 'next', 'self' and 'prev' links for the whole collection.
:param context: the current context, containing the original url path
and query string
:param refs: the list of members of the collection
:param hints: list hints, containing any relevant
filters. Any filters already satisfied by drivers
will have been removed
:param hints: list hints, containing any relevant filters and limit.
Any filters already satisfied by managers will have been
removed
"""
# Check if there are any filters in hints that were not
# handled by the drivers. The driver will not have paginated or
@ -287,7 +287,7 @@ class V3Controller(wsgi.Application):
if hints is not None:
refs = cls.filter_by_attributes(refs, hints)
refs = cls.paginate(context, refs)
list_limited, refs = cls.limit(refs, hints)
for ref in refs:
cls.wrap_member(context, ref)
@ -297,17 +297,45 @@ class V3Controller(wsgi.Application):
'next': None,
'self': cls.base_url(path=context['path']),
'previous': None}
if list_limited:
container['truncated'] = True
return container
@classmethod
def paginate(cls, context, refs):
"""Paginates a list of references by page & per_page query strings."""
# FIXME(dolph): client needs to support pagination first
return refs
def limit(cls, refs, hints):
"""Limits a list of entities.
page = context['query_string'].get('page', 1)
per_page = context['query_string'].get('per_page', 30)
return refs[per_page * (page - 1):per_page * page]
The underlying driver layer may have already truncated the collection
for us, but in case it was unable to handle truncation we check here.
:param refs: the list of members of the collection
:param hints: hints, containing, among other things, the limit
requested
:returns: boolean indicating whether the list was truncated, as well
as the list of (truncated if necessary) entities.
"""
NOT_LIMITED = False
LIMITED = True
if hints is None or hints.get_limit() is None:
# No truncation was requested
return NOT_LIMITED, refs
list_limit = hints.get_limit()
if list_limit.get('truncated', False):
# The driver did truncate the list
return LIMITED, refs
if len(refs) > list_limit['limit']:
# The driver layer wasn't able to truncate it for us, so we must
# do it here
return LIMITED, refs[:list_limit['limit']]
return NOT_LIMITED, refs
@classmethod
def filter_by_attributes(cls, refs, hints):

View File

@ -21,14 +21,15 @@ class Hints(list):
Hints are modifiers that affect the return of entities from a
list_<entities> operation. They are typically passed to a driver to give
direction as to what filtering and pagination actions are being requested.
direction as to what filtering, pagination or list limiting actions are
being requested.
It is optional for a driver to action some or all of the list hints,
but any filters that it does satisfy must be marked as such by calling
removing the filter from the list.
A Hint object is a list of dicts, initially all of type 'filter', although
other types may be added in the future. The list can be enumerated
A Hint object is a list of dicts, initially of type 'filter' or 'limit',
although other types may be added in the future. The list can be enumerated
directly, or by using the filters() method which will guarantee to only
return filters.
@ -60,3 +61,22 @@ class Hints(list):
if (entry['type'] == 'filter' and entry['name'] == name and
entry['comparator'] == 'equals'):
return entry
def set_limit(self, limit, truncated=False):
"""Set a limit to indicate the list should be truncated."""
# We only allow one limit entry in the list, so if it already exists
# we overwrite the old one
for x in self:
if x['type'] == 'limit':
x['limit'] = limit
x['truncated'] = truncated
break
else:
self.append({'limit': limit, 'type': 'limit',
'truncated': truncated})
def get_limit(self):
"""Get the limit to which the list should be truncated."""
for x in self:
if x['type'] == 'limit':
return x

View File

@ -19,6 +19,42 @@ import functools
from keystone.openstack.common import importutils
def response_truncated(f):
"""Truncate the list returned by the wrapped function.
This is designed to wrap Manager list_{entity} methods to ensure that
any list limits that are defined are passed to the driver layer. If a
hints list is provided, the wrapper will insert the relevant limit into
the hints so that the underlying driver call can try and honor it. If the
driver does truncate the response, it will update the 'truncated' attribute
in the 'limit' entry in the hints list, which enables the caller of this
function to know if truncation has taken place. If, however, the driver
layer is unable to perform truncation, the 'limit' entry is simply left in
the hints list for the caller to handle.
A _get_list_limit() method is required to be present in the object class
hierarchy, which returns the limit for this backend to which we will
truncate.
If a hints list is not provided in the arguments of the wrapped call then
any limits set in the config file are ignored. This allows internal use
of such wrapped methods where the entire data set is needed as input for
the calculations of some other API (e.g. get role assignments for a given
project).
"""
@functools.wraps(f)
def wrapper(self, *args, **kwargs):
if kwargs.get('hints') is None:
return f(self, *args, **kwargs)
list_limit = self.driver._get_list_limit()
if list_limit:
kwargs['hints'].set_limit(list_limit)
return f(self, *args, **kwargs)
return wrapper
class Manager(object):
"""Base class for intermediary request layer.

View File

@ -111,7 +111,7 @@ class XmlDeserializer(object):
values = {}
for k, v in six.iteritems(element.attrib):
# boolean-looking attributes become booleans in JSON
if k in ['enabled']:
if k in ['enabled', 'truncated']:
if v in ['true']:
v = True
elif v in ['false']:
@ -143,6 +143,7 @@ class XmlDeserializer(object):
return {'links': self._deserialize_links(element)}
links = None
truncated = False
for child in [self.walk_element(x) for x in element
if not isinstance(x, ENTITY_TYPE)]:
if list_item_tag:
@ -152,7 +153,10 @@ class XmlDeserializer(object):
if list_item_tag in child:
values.append(child[list_item_tag])
else:
links = child['links']
if 'links' in child:
links = child['links']
else:
truncated = child['truncated']
else:
values = dict(values.items() + child.items())
@ -167,6 +171,9 @@ class XmlDeserializer(object):
d['links'].setdefault('next')
d['links'].setdefault('previous')
if truncated:
d['truncated'] = truncated['truncated']
return d
@ -178,17 +185,23 @@ class XmlSerializer(object):
"""
links = None
truncated = False
# FIXME(dolph): skipping links for now
for key in d.keys():
if '_links' in key:
d.pop(key)
# FIXME(gyee): special-case links in collections
# NOTE(gyee, henry-nash): special-case links and truncation
# attribute in collections
if 'links' == key:
if links:
# we have multiple links
raise Exception('Multiple links found')
links = d.pop(key)
if 'truncated' == key:
if truncated:
# we have multiple attributes
raise Exception(_('Multiple truncation attributes found'))
truncated = d.pop(key)
assert len(d.keys()) == 1, ('Cannot encode more than one root '
'element: %s' % d.keys())
@ -206,9 +219,11 @@ class XmlSerializer(object):
self.populate_element(root, d[name])
# FIXME(gyee): special-case links for now
# NOTE(gyee, henry-nash): special-case links and truncation attribute
if links:
self._populate_links(root, links)
if truncated:
self._populate_truncated(root, truncated)
# TODO(dolph): you can get a doctype from lxml, using ElementTrees
return '%s\n%s' % (DOCTYPE, etree.tostring(root, pretty_print=True))
@ -223,6 +238,11 @@ class XmlSerializer(object):
links.append(link)
element.append(links)
def _populate_truncated(self, element, truncated_value):
truncated = etree.Element('truncated')
self._populate_bool(truncated, 'truncated', truncated_value)
element.append(truncated)
def _populate_list(self, element, k, v):
"""Populates an element with a key & list value."""
# spec has a lot of inconsistency here!

View File

@ -154,6 +154,47 @@ def transaction(expire_on_commit=False):
yield session
def truncated(f):
"""Ensure list truncation is detected in Driver list entity methods.
This is designed to wrap and sql Driver list_{entity} methods in order to
calculate if the resultant list has been truncated. Provided a limit dict
is found in the hints list, we increment the limit by one so as to ask the
wrapped function for one more entity than the limit, and then once the list
has been generated, we check to see if the original limit has been
exceeded, in which case we truncate back to that limit and set the
'truncated' boolean to 'true' in the hints limit dict.
"""
@functools.wraps(f)
def wrapper(self, *args, **kwargs):
if not hasattr(args[0], 'get_limit'):
raise exception.UnexpectedError(
_('Cannot truncate a driver call without hints list as '
'first parameter after self '))
hints = args[0]
limit_dict = hints.get_limit()
if limit_dict is None:
return f(self, *args, **kwargs)
# A limit is set, so ask for one more entry than we need
list_limit = limit_dict['limit']
hints.set_limit(list_limit + 1)
ref_list = f(self, *args, **kwargs)
# If we got more than the original limit then trim back the list and
# mark it truncated. In both cases, make sure we set the limit back
# to its original value.
if len(ref_list) > list_limit:
hints.set_limit(list_limit, truncated=True)
return ref_list[:list_limit]
else:
hints.set_limit(list_limit)
return ref_list
return wrapper
# Backends
class Base(object):
def _filter(self, model, query, hints):
@ -245,23 +286,56 @@ class Base(object):
return query
def filter_query(self, model, query, hints):
"""Applies filtering to a query.
def _limit(self, query, hints):
"""Applies a limit to a query.
:param model: table model
:param query: query to apply filters to
:param hints: contains the list of filters yet to be satisfied.
Any filters satisfied here will be removed so that
the caller will know if any filters remain.
:param hints: contains the list of filters and limit details.
:returns updated query
"""
if hints is not None:
query = self._filter(model, query, hints)
# NOTE(henry-nash): If we were to implement pagination, then we
# we would expand this method to support pagination and limiting.
# If we satisfied all the filters, set an upper limit if supplied
list_limit = hints.get_limit()
if list_limit:
query = query.limit(list_limit['limit'])
return query
def filter_limit_query(self, model, query, hints):
"""Applies filtering and limit to a query.
:param model: table model
:param query: query to apply filters to
:param hints: contains the list of filters and limit details. This may
be None, indicating that there are no filters or limits
to be applied. If it's not None, then any filters
satisfied here will be removed so that the caller will
know if any filters remain.
:returns: updated query
"""
if hints is None:
return query
# First try and satisfy any filters
query = self._filter(model, query, hints)
# NOTE(henry-nash): Any unsatisfied filters will have been left in
# the hints list for the controller to handle. We can only try and
# limit here if all the filters are already satisfied since, if not,
# doing so might mess up the final results. If there are still
# unsatisfied filters, we have to leave any limiting to the controller
# as well.
if not hints.filters():
return self._limit(query, hints)
else:
return query
def handle_conflicts(conflict_type='object'):
"""Converts select sqlalchemy exceptions into HTTP 409 Conflict."""

View File

@ -19,6 +19,7 @@ import json
from keystone.common import controller
from keystone.common import dependency
from keystone.common import driver_hints
from keystone import exception
@ -80,9 +81,13 @@ class CredentialV3(controller.V3Controller):
@controller.protected()
def list_credentials(self, context):
# NOTE(henry-nash): Since there are no filters for credentials, we
# shouldn't limit the output, hence we don't pass a hints list into
# the driver.
refs = self.credential_api.list_credentials()
ret_refs = [self._blob_to_json(r) for r in refs]
return CredentialV3.wrap_collection(context, ret_refs)
return CredentialV3.wrap_collection(context, ret_refs,
driver_hints.Hints())
@controller.protected()
def get_credential(self, context, credential_id):

View File

@ -124,10 +124,11 @@ class Identity(sql.Base, identity.Driver):
session.add(user_ref)
return identity.filter_user(user_ref.to_dict())
@sql.truncated
def list_users(self, hints):
session = db_session.get_session()
query = session.query(User)
user_refs = self.filter_query(User, query, hints)
user_refs = self.filter_limit_query(User, query, hints)
return [identity.filter_user(x.to_dict()) for x in user_refs]
def _get_user(self, session, user_id):
@ -256,10 +257,11 @@ class Identity(sql.Base, identity.Driver):
session.add(ref)
return ref.to_dict()
@sql.truncated
def list_groups(self, hints):
session = db_session.get_session()
query = session.query(Group)
refs = self.filter_query(Group, query, hints)
refs = self.filter_limit_query(Group, query, hints)
return [ref.to_dict() for ref in refs]
def _get_group(self, session, group_id):

View File

@ -363,6 +363,7 @@ class Manager(manager.Manager):
ref = self._set_domain_id(ref, domain_id)
return ref
@manager.response_truncated
@domains_configured
def list_users(self, domain_scope=None, hints=None):
domain_id, driver = self._get_domain_id_and_driver(domain_scope)
@ -463,6 +464,7 @@ class Manager(manager.Manager):
driver.remove_user_from_group(user_id, group_id)
self.token_api.delete_tokens_for_user(user_id)
@manager.response_truncated
@domains_configured
def list_groups_for_user(self, user_id, domain_scope=None,
hints=None):
@ -477,6 +479,7 @@ class Manager(manager.Manager):
ref_list = self._set_domain_id(ref_list, domain_id)
return ref_list
@manager.response_truncated
@domains_configured
def list_groups(self, domain_scope=None, hints=None):
domain_id, driver = self._get_domain_id_and_driver(domain_scope)
@ -489,6 +492,7 @@ class Manager(manager.Manager):
ref_list = self._set_domain_id(ref_list, domain_id)
return ref_list
@manager.response_truncated
@domains_configured
def list_users_in_group(self, group_id, domain_scope=None,
hints=None):
@ -638,6 +642,9 @@ class Manager(manager.Manager):
class Driver(object):
"""Interface description for an Identity driver."""
def _get_list_limit(self):
return CONF.identity.list_limit or CONF.list_limit
@abc.abstractmethod
def authenticate(self, user_id, password):
"""Authenticate a given user and password.

View File

@ -35,10 +35,7 @@ class PolicyV3(controller.V3Controller):
@controller.filterprotected('type')
def list_policies(self, context, filters):
hints = PolicyV3.build_driver_hints(context, filters)
# We don't bother passing the hints in, since this would be
# a highly unlikely filter to use - wrap_collection() can
# handle if required.
refs = self.policy_api.list_policies()
refs = self.policy_api.list_policies(hints=hints)
return PolicyV3.wrap_collection(context, refs, hints=hints)
@controller.protected()

View File

@ -55,6 +55,13 @@ class Manager(manager.Manager):
except exception.NotFound:
raise exception.PolicyNotFound(policy_id=policy_id)
@manager.response_truncated
def list_policies(self, hints=None):
# NOTE(henry-nash): Since the advantage of filtering or list limiting
# of policies at the driver level is minimal, we leave this to the
# caller.
return self.driver.list_policies()
def delete_policy(self, policy_id):
try:
return self.driver.delete_policy(policy_id)
@ -65,6 +72,9 @@ class Manager(manager.Manager):
@six.add_metaclass(abc.ABCMeta)
class Driver(object):
def _get_list_limit(self):
return CONF.policy.list_limit or CONF.list_limit
@abc.abstractmethod
def enforce(self, context, credentials, action, target):
"""Verify that a user is authorized to perform action.

View File

@ -4093,3 +4093,76 @@ class FilterTests(filtering.FilterTests):
groups = self.identity_api.list_groups()
self.assertTrue(len(groups) > 0)
class LimitTests(filtering.FilterTests):
def setUp(self):
"""Setup for Limit Test Cases."""
self.domain1 = {'id': uuid.uuid4().hex, 'name': uuid.uuid4().hex}
self.assignment_api.create_domain(self.domain1['id'], self.domain1)
self.addCleanup(self.clean_up_domain)
self.entity_lists = {}
self.domain1_entity_lists = {}
for entity in ['user', 'group', 'project']:
# Create 20 entities, 14 of which are in domain1
self.entity_lists[entity] = self._create_test_data(entity, 6)
self.domain1_entity_lists[entity] = self._create_test_data(
entity, 14, self.domain1['id'])
# Make sure we clean up when finished
self.addCleanup(self.clean_up_entity, entity)
def clean_up_domain(self):
"""Clean up domain test data from Limit Test Cases."""
self.domain1['enabled'] = False
self.assignment_api.update_domain(self.domain1['id'], self.domain1)
self.assignment_api.delete_domain(self.domain1['id'])
def clean_up_entity(self, entity):
"""Clean up entity test data from Limit Test Cases."""
self._delete_test_data(entity, self.entity_lists[entity])
self._delete_test_data(entity, self.domain1_entity_lists[entity])
def _test_list_entity_filtered_and_limited(self, entity):
self.opt(list_limit=10)
# Should get back just 10 entities in domain1
hints = driver_hints.Hints()
hints.add_filter('domain_id', self.domain1['id'])
entities = self._list_entities(entity)(hints=hints)
self.assertEqual(len(entities), hints.get_limit()['limit'])
self.assertTrue(hints.get_limit()['truncated'])
self._match_with_list(entities, self.domain1_entity_lists[entity])
# Override with driver specific limit
if entity == 'project':
self.opt_in_group('assignment', list_limit=5)
else:
self.opt_in_group('identity', list_limit=5)
# Should get back just 5 users in domain1
hints = driver_hints.Hints()
hints.add_filter('domain_id', self.domain1['id'])
entities = self._list_entities(entity)(hints=hints)
self.assertEqual(len(entities), hints.get_limit()['limit'])
self._match_with_list(entities, self.domain1_entity_lists[entity])
# Finally, let's pretend we want to get the full list of entities,
# even with the limits set, as part of some internal calculation.
# Calling the API without a hints list should achieve this, and
# return at least the 20 entries we created (there may be other
# entities lying around created by other tests/setup).
entities = self._list_entities(entity)()
self.assertTrue(len(entities) >= 20)
def test_list_users_filtered_and_limited(self):
self._test_list_entity_filtered_and_limited('user')
def test_list_groups_filtered_and_limited(self):
self._test_list_entity_filtered_and_limited('group')
def test_list_projects_filtered_and_limited(self):
self._test_list_entity_filtered_and_limited('project')

View File

@ -486,3 +486,9 @@ class SqlTokenCacheInvalidation(SqlTests, test_backend.TokenCacheInvalidation):
class SqlFilterTests(SqlTests, test_backend.FilterTests):
pass
class SqlLimitTests(SqlTests, test_backend.LimitTests):
def setUp(self):
super(SqlLimitTests, self).setUp()
test_backend.LimitTests.setUp(self)

View File

@ -49,3 +49,16 @@ class ListHintsTests(test.TestCase):
hints2.add_filter('t4', 'data1')
hints2.add_filter('t5', 'data2')
self.assertEqual(len(hints.filters()), 2)
def test_limits(self):
hints = driver_hints.Hints()
self.assertIsNone(hints.get_limit())
hints.set_limit(10)
self.assertEqual(hints.get_limit()['limit'], 10)
self.assertFalse(hints.get_limit()['truncated'])
hints.set_limit(11)
self.assertEqual(hints.get_limit()['limit'], 11)
self.assertFalse(hints.get_limit()['truncated'])
hints.set_limit(10, truncated=True)
self.assertEqual(hints.get_limit()['limit'], 10)
self.assertTrue(hints.get_limit()['truncated'])

View File

@ -31,6 +31,7 @@ CONF = config.CONF
class IdentityTestFilteredCase(filtering.FilterTests,
test_v3.RestfulTestCase):
"""Test filter enforcement on the v3 Identity API."""
content_type = 'json'
def setUp(self):
"""Setup for Identity Filter Test Cases."""
@ -299,3 +300,141 @@ class IdentityTestFilteredCase(filtering.FilterTests,
url_by_name = "/groups"
r = self.get(url_by_name, auth=self.auth)
self.assertTrue(len(r.result.get('groups')) > 0)
class IdentityTestListLimitCase(IdentityTestFilteredCase):
"""Test list limiting enforcement on the v3 Identity API."""
content_type = 'json'
def setUp(self):
"""Setup for Identity Limit Test Cases."""
super(IdentityTestListLimitCase, self).setUp()
self._set_policy({"identity:list_users": [],
"identity:list_groups": [],
"identity:list_projects": [],
"identity:list_services": [],
"identity:list_policies": []})
# Create 10 entries for each of the entities we are going to test
self.ENTITY_TYPES = ['user', 'group', 'project']
self.entity_lists = {}
for entity in self.ENTITY_TYPES:
self.entity_lists[entity] = self._create_test_data(entity, 10)
# Make sure we clean up when finished
self.addCleanup(self.clean_up_entity, entity)
self.service_list = []
self.addCleanup(self.clean_up_service)
for _ in range(10):
new_entity = {'id': uuid.uuid4().hex, 'type': uuid.uuid4().hex}
service = self.catalog_api.create_service(new_entity['id'],
new_entity)
self.service_list.append(service)
self.policy_list = []
self.addCleanup(self.clean_up_policy)
for _ in range(10):
new_entity = {'id': uuid.uuid4().hex, 'type': uuid.uuid4().hex,
'blob': uuid.uuid4().hex}
policy = self.policy_api.create_policy(new_entity['id'],
new_entity)
self.policy_list.append(policy)
def clean_up_entity(self, entity):
"""Clean up entity test data from Identity Limit Test Cases."""
self._delete_test_data(entity, self.entity_lists[entity])
def clean_up_service(self):
"""Clean up service test data from Identity Limit Test Cases."""
for service in self.service_list:
self.catalog_api.delete_service(service['id'])
def clean_up_policy(self):
"""Clean up policy test data from Identity Limit Test Cases."""
for policy in self.policy_list:
self.policy_api.delete_policy(policy['id'])
def _test_entity_list_limit(self, entity, driver):
"""GET /<entities> (limited)
Test Plan:
- For the specified type of entity:
- Update policy for no protection on api
- Add a bunch of entities
- Set the global list limit to 5, and check that getting all
- entities only returns 5
- Set the driver list_limit to 4, and check that now only 4 are
- returned
"""
if entity == 'policy':
plural = 'policies'
else:
plural = '%ss' % entity
self.opt(list_limit=5)
self.opt_in_group(driver, list_limit=None)
r = self.get('/%s' % plural, auth=self.auth)
self.assertEqual(len(r.result.get(plural)), 5)
self.assertIs(r.result.get('truncated'), True)
self.opt_in_group(driver, list_limit=4)
r = self.get('/%s' % plural, auth=self.auth)
self.assertEqual(len(r.result.get(plural)), 4)
self.assertIs(r.result.get('truncated'), True)
def test_users_list_limit(self):
self._test_entity_list_limit('user', 'identity')
def test_groups_list_limit(self):
self._test_entity_list_limit('group', 'identity')
def test_projects_list_limit(self):
self._test_entity_list_limit('project', 'assignment')
def test_services_list_limit(self):
self._test_entity_list_limit('service', 'catalog')
def test_non_driver_list_limit(self):
"""Check list can be limited without driver level support.
Policy limiting is not done at the driver level (since it
really isn't worth doing it there). So use this as a test
for ensuring the controller level will successfully limit
in this case.
"""
self._test_entity_list_limit('policy', 'policy')
def test_no_limit(self):
"""Check truncated attribute not set when list not limited."""
r = self.get('/services', auth=self.auth)
self.assertEqual(len(r.result.get('services')), 10)
self.assertIsNone(r.result.get('truncated'))
def test_at_limit(self):
"""Check truncated attribute not set when list at max size."""
# Test this by overriding the general limit with a higher
# driver-specific limit (allowing all entities to be returned
# in the collection), which should result in a non truncated list
self.opt(list_limit=5)
self.opt_in_group('catalog', list_limit=10)
r = self.get('/services', auth=self.auth)
self.assertEqual(len(r.result.get('services')), 10)
self.assertIsNone(r.result.get('truncated'))
class IdentityTestFilteredCaseXML(IdentityTestFilteredCase):
content_type = 'xml'
class IdentityTestListLimitCaseXML(IdentityTestListLimitCase):
content_type = 'xml'