Implement basic caching around assignment CRUD

Implements caching around basic assignment CRUD actions.
    * assignment_api.get_domain
    * assignmnet_api.get_domain_by_name
    * assignment_api.get_project
    * assignment_api.get_project_by_name
    * assignment_api.get_role

The Create, Update, and Delete actions for domains, projects and
roles will perform proper invalidations of the cached methods listed
above.

Specific Cache Layer Tests added around assignment CRUD.

Modifications of LDAP tests done to handle caching concepts.

List methods are not covered by this patchset (list_projects, list_domains,
list_roles).

Grants are not subject to caching in this patchset.

DocImpact

partial-blueprint: caching-layer-for-driver-calls
Change-Id: Ic4e1a2d93078e55649ce9410ebece9b4d09b095a
This commit is contained in:
Morgan Fainberg 2013-08-24 21:07:32 -07:00
parent e587957faf
commit ead4f98e82
7 changed files with 388 additions and 7 deletions

View File

@ -264,6 +264,28 @@ Current keystone systems that have caching capabilities:
``revocation_cache_time`` in the ``[token]`` section. The revocation
list is refreshed whenever a token is revoked. It typically sees significantly
more requests than specific token retrievals or token validation calls.
* ``assignment``
The assignment system has a separate ``cache_time`` configuration option,
that can be set to a value above or below the global ``expiration_time``
default, allowing for different caching behavior from the other systems in
``Keystone``. This option is set in the ``[assignment]`` section of the
configuration file.
Currently ``assignment`` has caching for ``project``, ``domain``, and ``role``
specific requests (primarily around the CRUD actions). Caching is currently not
implemented on grants. The list (``list_projects``, ``list_domains``, etc)
methods are not subject to caching.
.. WARNING::
Be aware that if a read-only ``assignment`` backend is in use, the cache
will not immediately reflect changes on the back end. Any given change
may take up to the ``cache_time`` (if set in the ``[assignment]``
section of the configuration) or the global ``expiration_time`` (set in
the ``[cache]`` section of the configuration) before it is reflected.
If this type of delay (when using a read-only ``assignment`` backend) is
an issue, it is recommended that caching be disabled on ``assignment``.
To disable caching specifically on ``assignment``, in the ``[assignment]``
section of the configuration set ``caching`` to ``False``.
For more information about the different backends (and configuration options):
* `dogpile.cache.backends.memory`_

View File

@ -220,6 +220,13 @@
[assignment]
# driver =
# Assignment specific caching toggle. This has no effect unless the global
# caching option is set to True
# caching = True
# Assignment specific cache time-to-live (TTL) in seconds.
# cache_time =
[oauth1]
# driver = keystone.contrib.oauth1.backends.sql.OAuth1

View File

@ -16,7 +16,9 @@
"""Main entry point into the assignment service."""
from keystone import clean
from keystone.common import cache
from keystone.common import dependency
from keystone.common import manager
from keystone import config
@ -27,6 +29,7 @@ from keystone.openstack.common import log as logging
CONF = config.CONF
LOG = logging.getLogger(__name__)
SHOULD_CACHE = cache.should_cache_fn('assignment')
DEFAULT_DOMAIN = {'description':
(u'Owns users and tenants (i.e. projects)'
@ -63,18 +66,32 @@ class Manager(manager.Manager):
tenant.setdefault('enabled', True)
tenant['enabled'] = clean.project_enabled(tenant['enabled'])
tenant.setdefault('description', '')
return self.driver.create_project(tenant_id, tenant)
ret = self.driver.create_project(tenant_id, tenant_ref)
if SHOULD_CACHE(ret):
self.get_project.set(ret, self, tenant_id)
self.get_project_by_name.set(ret, self, ret['name'],
ret['domain_id'])
return ret
@notifications.updated('project')
def update_project(self, tenant_id, tenant_ref):
tenant = tenant_ref.copy()
if 'enabled' in tenant:
tenant['enabled'] = clean.project_enabled(tenant['enabled'])
return self.driver.update_project(tenant_id, tenant)
ret = self.driver.update_project(tenant_id, tenant_ref)
self.get_project.invalidate(self, tenant_id)
self.get_project_by_name.invalidate(self, ret['name'],
ret['domain_id'])
return ret
@notifications.deleted('project')
def delete_project(self, tenant_id):
return self.driver.delete_project(tenant_id)
project = self.driver.get_project(tenant_id)
ret = self.driver.delete_project(tenant_id)
self.get_project.invalidate(self, tenant_id)
self.get_project_by_name.invalidate(self, project['name'],
project['domain_id'])
return ret
def get_roles_for_user_and_project(self, user_id, tenant_id):
"""Get the roles associated with a user within given project.
@ -226,6 +243,65 @@ class Manager(manager.Manager):
for role_id in roles:
self.remove_role_from_user_and_project(user_id, tenant_id, role_id)
@cache.on_arguments(should_cache_fn=SHOULD_CACHE,
expiration_time=CONF.assignment.cache_time)
def get_domain(self, domain_id):
return self.driver.get_domain(domain_id)
@cache.on_arguments(should_cache_fn=SHOULD_CACHE,
expiration_time=CONF.assignment.cache_time)
def get_domain_by_name(self, domain_name):
return self.driver.get_domain_by_name(domain_name)
def create_domain(self, domain_id, domain):
ret = self.driver.create_domain(domain_id, domain)
if SHOULD_CACHE(ret):
self.get_domain.set(ret, self, domain_id)
self.get_domain_by_name.set(ret, self, ret['name'])
return ret
def update_domain(self, domain_id, domain):
ret = self.driver.update_domain(domain_id, domain)
self.get_domain.invalidate(self, domain_id)
self.get_domain_by_name.invalidate(self, ret['name'])
return ret
def delete_domain(self, domain_id):
domain = self.driver.get_domain(domain_id)
self.driver.delete_domain(domain_id)
self.get_domain.invalidate(self, domain_id)
self.get_domain_by_name.invalidate(self, domain['name'])
@cache.on_arguments(should_cache_fn=SHOULD_CACHE,
expiration_time=CONF.assignment.cache_time)
def get_project(self, project_id):
return self.driver.get_project(project_id)
@cache.on_arguments(should_cache_fn=SHOULD_CACHE,
expiration_time=CONF.assignment.cache_time)
def get_project_by_name(self, tenant_name, domain_id):
return self.driver.get_project_by_name(tenant_name, domain_id)
@cache.on_arguments(should_cache_fn=SHOULD_CACHE,
expiration_time=CONF.assignment.cache_time)
def get_role(self, role_id):
return self.driver.get_role(role_id)
def create_role(self, role_id, role):
ret = self.driver.create_role(role_id, role)
if SHOULD_CACHE(ret):
self.get_role.set(ret, self, role_id)
return ret
def update_role(self, role_id, role):
ret = self.driver.update_role(role_id, role)
self.get_role.invalidate(self, role_id)
return ret
def delete_role(self, role_id):
self.driver.delete_role(role_id)
self.get_role.invalidate(self, role_id)
class Driver(object):

View File

@ -59,15 +59,15 @@ class DebugProxy(proxy.ProxyBackend):
return self.proxied.set(key, value)
def set_multi(self, keys):
LOG.debug(_('CACHE_SET_MULTI: %s') % keys)
LOG.debug(_('CACHE_SET_MULTI: "%s"') % keys)
self.proxied.set_multi(keys)
def delete(self, key):
self.proxied.delete(key)
LOG.debug(_('CACHE_DELETE: %s') % key)
LOG.debug(_('CACHE_DELETE: "%s"') % key)
def delete_multi(self, keys):
LOG.debug(_('CACHE_DELETE_MULTI: %s') % keys)
LOG.debug(_('CACHE_DELETE_MULTI: "%s"') % keys)
self.proxied.delete_multi(keys)

View File

@ -133,7 +133,9 @@ FILE_OPTIONS = {
# assignment has no default for backward compatibility reasons.
# If assignment driver is not specified, the identity driver chooses
# the backend
cfg.StrOpt('driver', default=None)],
cfg.StrOpt('driver', default=None),
cfg.BoolOpt('caching', default=True),
cfg.IntOpt('cache_time', default=None)],
'credential': [
cfg.StrOpt('driver',
default=('keystone.credential.backends'

View File

@ -14,6 +14,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import copy
import datetime
import uuid
@ -2277,6 +2278,146 @@ class IdentityTests(object):
user_projects = self.identity_api.list_user_projects(user1['id'])
self.assertEquals(len(user_projects), 2)
def test_cache_layer_domain_crud(self):
domain = {'id': uuid.uuid4().hex, 'name': uuid.uuid4().hex,
'enabled': True}
domain_id = domain['id']
# Create Domain
self.assignment_api.create_domain(domain_id, domain)
domain_ref = self.assignment_api.get_domain(domain_id)
updated_domain_ref = copy.deepcopy(domain_ref)
updated_domain_ref['name'] = uuid.uuid4().hex
# Update domain, bypassing assignment api manager
self.assignment_api.driver.update_domain(domain_id, updated_domain_ref)
# Verify get_domain still returns the domain
self.assertDictContainsSubset(
domain_ref, self.assignment_api.get_domain(domain_id))
# Invalidate cache
self.assignment_api.get_domain.invalidate(self.assignment_api,
domain_id)
# Verify get_domain returns the updated domain
self.assertDictContainsSubset(
updated_domain_ref, self.assignment_api.get_domain(domain_id))
# Update the domain back to original ref, using the assignment api
# manager
self.assignment_api.update_domain(domain_id, domain_ref)
self.assertDictContainsSubset(
domain_ref, self.assignment_api.get_domain(domain_id))
# Delete domain, bypassing assignment api manager
self.assignment_api.driver.delete_domain(domain_id)
# Verify get_domain still returns the domain
self.assertDictContainsSubset(
domain_ref, self.assignment_api.get_domain(domain_id))
# Invalidate cache
self.assignment_api.get_domain.invalidate(self.assignment_api,
domain_id)
# Verify get_domain now raises DomainNotFound
self.assertRaises(exception.DomainNotFound,
self.assignment_api.get_domain, domain_id)
# Recreate Domain
self.identity_api.create_domain(domain_id, domain)
self.assignment_api.get_domain(domain_id)
# Delete domain
self.assignment_api.delete_domain(domain_id)
# verify DomainNotFound raised
self.assertRaises(exception.DomainNotFound,
self.assignment_api.get_domain,
domain_id)
def test_cache_layer_project_crud(self):
domain = {'id': uuid.uuid4().hex, 'name': uuid.uuid4().hex,
'enabled': True}
project = {'id': uuid.uuid4().hex, 'name': uuid.uuid4().hex,
'domain_id': domain['id']}
project_id = project['id']
self.assignment_api.create_domain(domain['id'], domain)
# Create a project
self.assignment_api.create_project(project_id, project)
self.assignment_api.get_project(project_id)
updated_project = copy.deepcopy(project)
updated_project['name'] = uuid.uuid4().hex
# Update project, bypassing assignment_api manager
self.assignment_api.driver.update_project(project_id,
updated_project)
# Verify get_project still returns the original project_ref
self.assertDictContainsSubset(
project, self.assignment_api.get_project(project_id))
# Invalidate cache
self.assignment_api.get_project.invalidate(self.assignment_api,
project_id)
# Verify get_project now returns the new project
self.assertDictContainsSubset(
updated_project,
self.assignment_api.get_project(project_id))
# Update project using the assignment_api manager back to original
self.assignment_api.update_project(project['id'], project)
# Verify get_project returns the original project_ref
self.assertDictContainsSubset(
project, self.assignment_api.get_project(project_id))
# Delete project bypassing assignment_api
self.assignment_api.driver.delete_project(project_id)
# Verify get_project still returns the project_ref
self.assertDictContainsSubset(
project, self.assignment_api.get_project(project_id))
# Invalidate cache
self.assignment_api.get_project.invalidate(self.assignment_api,
project_id)
# Verify ProjectNotFound now raised
self.assertRaises(exception.ProjectNotFound,
self.assignment_api.get_project,
project_id)
# recreate project
self.assignment_api.create_project(project_id, project)
self.assignment_api.get_project(project_id)
# delete project
self.assignment_api.delete_project(project_id)
# Verify ProjectNotFound is raised
self.assertRaises(exception.ProjectNotFound,
self.assignment_api.get_project,
project_id)
def test_cache_layer_role_crud(self):
role = {'id': uuid.uuid4().hex, 'name': uuid.uuid4().hex}
role_id = role['id']
# Create role
self.assignment_api.create_role(role_id, role)
role_ref = self.assignment_api.get_role(role_id)
updated_role_ref = copy.deepcopy(role_ref)
updated_role_ref['name'] = uuid.uuid4().hex
# Update role, bypassing the assignment api manager
self.assignment_api.driver.update_role(role_id, updated_role_ref)
# Verify get_role still returns old ref
self.assertDictEqual(role_ref, self.assignment_api.get_role(role_id))
# Invalidate Cache
self.assignment_api.get_role.invalidate(self.assignment_api,
role_id)
# Verify get_role returns the new role_ref
self.assertDictEqual(updated_role_ref,
self.assignment_api.get_role(role_id))
# Update role back to original via the assignment api manager
self.assignment_api.update_role(role_id, role_ref)
# Verify get_role returns the original role ref
self.assertDictEqual(role_ref, self.assignment_api.get_role(role_id))
# Delete role bypassing the assignment api manager
self.assignment_api.driver.delete_role(role_id)
# Verify get_role still returns the role_ref
self.assertDictEqual(role_ref, self.assignment_api.get_role(role_id))
# Invalidate cache
self.assignment_api.get_role.invalidate(self.assignment_api, role_id)
# Verify RoleNotFound is now raised
self.assertRaises(exception.RoleNotFound,
self.assignment_api.get_role,
role_id)
# recreate role
self.assignment_api.create_role(role_id, role)
self.assignment_api.get_role(role_id)
# delete role via the assignment api manager
self.assignment_api.delete_role(role_id)
# verity RoleNotFound is now raised
self.assertRaises(exception.RoleNotFound,
self.assignment_api.get_role,
role_id)
class TokenTests(object):
def _create_token_id(self):

View File

@ -15,11 +15,13 @@
# License for the specific language governing permissions and limitations
# under the License.
import copy
import uuid
import ldap
from keystone import assignment
from keystone.common import cache
from keystone.common.ldap import fakeldap
from keystone.common import sql
from keystone import config
@ -390,6 +392,17 @@ class LDAPIdentity(test.TestCase, BaseLDAPIdentity):
CONF.ldap.tenant_filter = '(CN=DOES_NOT_MATCH)'
self.load_backends()
# NOTE(morganfainberg): CONF.ldap.tenant_filter will not be
# dynamically changed at runtime. This invalidate is a work-around for
# the expectation that it is safe to change config values in tests that
# could affect what the drivers would return up to the manager. This
# solves this assumption when working with aggressive (on-create)
# cache population.
self.assignment_api.get_role.invalidate(self.assignment_api,
self.role_member['id'])
self.identity_api.get_role(self.role_member['id'])
self.assignment_api.get_project.invalidate(self.assignment_api,
self.tenant_bar['id'])
self.assertRaises(exception.ProjectNotFound,
self.identity_api.get_project,
self.tenant_bar['id'])
@ -400,6 +413,14 @@ class LDAPIdentity(test.TestCase, BaseLDAPIdentity):
CONF.ldap.role_filter = '(CN=DOES_NOT_MATCH)'
self.load_backends()
# NOTE(morganfainberg): CONF.ldap.role_filter will not be
# dynamically changed at runtime. This invalidate is a work-around for
# the expectation that it is safe to change config values in tests that
# could affect what the drivers would return up to the manager. This
# solves this assumption when working with aggressive (on-create)
# cache population.
self.assignment_api.get_role.invalidate(self.assignment_api,
self.role_member['id'])
self.assertRaises(exception.RoleNotFound,
self.identity_api.get_role,
self.role_member['id'])
@ -421,6 +442,16 @@ class LDAPIdentity(test.TestCase, BaseLDAPIdentity):
self.clear_database()
self.load_backends()
self.load_fixtures(default_fixtures)
# NOTE(morganfainberg): CONF.ldap.tenant_name_attribute,
# CONF.ldap.tenant_desc_attribute, and
# CONF.ldap.tenant_enabled_attribute will not be
# dynamically changed at runtime. This invalidate is a work-around for
# the expectation that it is safe to change config values in tests that
# could affect what the drivers would return up to the manager. This
# solves this assumption when working with aggressive (on-create)
# cache population.
self.assignment_api.get_project.invalidate(self.assignment_api,
self.tenant_baz['id'])
tenant_ref = self.identity_api.get_project(self.tenant_baz['id'])
self.assertEqual(tenant_ref['id'], self.tenant_baz['id'])
self.assertEqual(tenant_ref['name'], self.tenant_baz['name'])
@ -432,6 +463,16 @@ class LDAPIdentity(test.TestCase, BaseLDAPIdentity):
CONF.ldap.tenant_name_attribute = 'description'
CONF.ldap.tenant_desc_attribute = 'ou'
self.load_backends()
# NOTE(morganfainberg): CONF.ldap.tenant_name_attribute,
# CONF.ldap.tenant_desc_attribute, and
# CONF.ldap.tenant_enabled_attribute will not be
# dynamically changed at runtime. This invalidate is a work-around for
# the expectation that it is safe to change config values in tests that
# could affect what the drivers would return up to the manager. This
# solves this assumption when working with aggressive (on-create)
# cache population.
self.assignment_api.get_project.invalidate(self.assignment_api,
self.tenant_baz['id'])
tenant_ref = self.identity_api.get_project(self.tenant_baz['id'])
self.assertEqual(tenant_ref['id'], self.tenant_baz['id'])
self.assertEqual(tenant_ref['name'], self.tenant_baz['description'])
@ -445,6 +486,14 @@ class LDAPIdentity(test.TestCase, BaseLDAPIdentity):
self.clear_database()
self.load_backends()
self.load_fixtures(default_fixtures)
# NOTE(morganfainberg): CONF.ldap.tenant_attribute_ignore will not be
# dynamically changed at runtime. This invalidate is a work-around for
# the expectation that it is safe to change configs values in tests
# that could affect what the drivers would return up to the manager.
# This solves this assumption when working with aggressive (on-create)
# cache population.
self.assignment_api.get_project.invalidate(self.assignment_api,
self.tenant_baz['id'])
tenant_ref = self.identity_api.get_project(self.tenant_baz['id'])
self.assertEqual(tenant_ref['id'], self.tenant_baz['id'])
self.assertNotIn('name', tenant_ref)
@ -456,12 +505,28 @@ class LDAPIdentity(test.TestCase, BaseLDAPIdentity):
self.clear_database()
self.load_backends()
self.load_fixtures(default_fixtures)
# NOTE(morganfainberg): CONF.ldap.role_name_attribute will not be
# dynamically changed at runtime. This invalidate is a work-around for
# the expectation that it is safe to change config values in tests that
# could affect what the drivers would return up to the manager. This
# solves this assumption when working with aggressive (on-create)
# cache population.
self.assignment_api.get_role.invalidate(self.assignment_api,
self.role_member['id'])
role_ref = self.identity_api.get_role(self.role_member['id'])
self.assertEqual(role_ref['id'], self.role_member['id'])
self.assertEqual(role_ref['name'], self.role_member['name'])
CONF.ldap.role_name_attribute = 'sn'
self.load_backends()
# NOTE(morganfainberg): CONF.ldap.role_name_attribute will not be
# dynamically changed at runtime. This invalidate is a work-around for
# the expectation that it is safe to change config values in tests that
# could affect what the drivers would return up to the manager. This
# solves this assumption when working with aggressive (on-create)
# cache population.
self.assignment_api.get_role.invalidate(self.assignment_api,
self.role_member['id'])
role_ref = self.identity_api.get_role(self.role_member['id'])
self.assertEqual(role_ref['id'], self.role_member['id'])
self.assertNotIn('name', role_ref)
@ -471,6 +536,14 @@ class LDAPIdentity(test.TestCase, BaseLDAPIdentity):
self.clear_database()
self.load_backends()
self.load_fixtures(default_fixtures)
# NOTE(morganfainberg): CONF.ldap.role_attribute_ignore will not be
# dynamically changed at runtime. This invalidate is a work-around for
# the expectation that it is safe to change config values in tests that
# could affect what the drivers would return up to the manager. This
# solves this assumption when working with aggressive (on-create)
# cache population.
self.assignment_api.get_role.invalidate(self.assignment_api,
self.role_member['id'])
role_ref = self.identity_api.get_role(self.role_member['id'])
self.assertEqual(role_ref['id'], self.role_member['id'])
self.assertNotIn('name', role_ref)
@ -618,6 +691,12 @@ class LDAPIdentity(test.TestCase, BaseLDAPIdentity):
self.identity_api.get_domain,
domain['id'])
def test_cache_layer_domain_crud(self):
# TODO(morganfainberg): This also needs to be removed when full LDAP
# implementation is submitted. No need to duplicate the above test,
# just skip this time.
self.skipTest('Domains are read-only against LDAP')
def test_project_crud(self):
# NOTE(topol): LDAP implementation does not currently support the
# updating of a project name so this method override
@ -641,6 +720,59 @@ class LDAPIdentity(test.TestCase, BaseLDAPIdentity):
self.identity_api.get_project,
project['id'])
def test_cache_layer_project_crud(self):
# NOTE(morganfainberg): LDAP implementation does not currently support
# updating project names. This method override provides a different
# update test.
project = {'id': uuid.uuid4().hex, 'name': uuid.uuid4().hex,
'domain_id': CONF.identity.default_domain_id,
'description': uuid.uuid4().hex}
project_id = project['id']
# Create a project
self.assignment_api.create_project(project_id, project)
self.assignment_api.get_project(project_id)
updated_project = copy.deepcopy(project)
updated_project['description'] = uuid.uuid4().hex
# Update project, bypassing assignment_api manager
self.assignment_api.driver.update_project(project_id,
updated_project)
# Verify get_project still returns the original project_ref
self.assertDictContainsSubset(
project, self.assignment_api.get_project(project_id))
# Invalidate cache
self.assignment_api.get_project.invalidate(self.assignment_api,
project_id)
# Verify get_project now returns the new project
self.assertDictContainsSubset(
updated_project,
self.assignment_api.get_project(project_id))
# Update project using the assignment_api manager back to original
self.assignment_api.update_project(project['id'], project)
# Verify get_project returns the original project_ref
self.assertDictContainsSubset(
project, self.assignment_api.get_project(project_id))
# Delete project bypassing assignment_api
self.assignment_api.driver.delete_project(project_id)
# Verify get_project still returns the project_ref
self.assertDictContainsSubset(
project, self.assignment_api.get_project(project_id))
# Invalidate cache
self.assignment_api.get_project.invalidate(self.assignment_api,
project_id)
# Verify ProjectNotFound now raised
self.assertRaises(exception.ProjectNotFound,
self.assignment_api.get_project,
project_id)
# recreate project
self.assignment_api.create_project(project_id, project)
self.assignment_api.get_project(project_id)
# delete project
self.assignment_api.delete_project(project_id)
# Verify ProjectNotFound is raised
self.assertRaises(exception.ProjectNotFound,
self.assignment_api.get_project,
project_id)
def test_multi_role_grant_by_user_group_on_project_domain(self):
# This is a partial implementation of the standard test that
# is defined in test_backend.py. It omits both domain and
@ -781,6 +913,7 @@ class LdapIdentitySqlAssignment(sql.Base, test.TestCase, BaseLDAPIdentity):
self._set_config()
self.clear_database()
self.load_backends()
cache.configure_cache_region(cache.REGION)
self.engine = self.get_engine()
sql.ModelBase.metadata.create_all(bind=self.engine)
self.load_fixtures(default_fixtures)