From 4969f66fca37ce9bd654cf74890fa28dd011bc6e Mon Sep 17 00:00:00 2001 From: Steve Martinelli Date: Fri, 26 Jun 2015 01:33:24 -0400 Subject: [PATCH] switch to oslo.cache Use oslo.cache now that it's been made available. Note that we only deprecate the cache backends, but not remove them, they are public APIs and subject to deprecation. Also remove most of the caching code, but keep the portions that register backends, since they are still available. Note that the tests that were deleted were testing cache functionality, they now exist in oslo.cache. Co-Authored-By: Brant Knudson Change-Id: I39d3ff1dd2edad79effa56450b444174b09f971e --- config-generator/keystone.conf | 1 + keystone/assignment/core.py | 2 +- keystone/catalog/core.py | 2 +- .../common/cache/backends/memcache_pool.py | 53 +- keystone/common/cache/backends/mongo.py | 554 +------------ keystone/common/cache/backends/noop.py | 6 + keystone/common/cache/core.py | 284 +------ keystone/common/config.py | 79 +- keystone/common/kvs/backends/memcached.py | 2 +- keystone/contrib/revoke/core.py | 2 +- keystone/identity/core.py | 2 +- keystone/resource/core.py | 4 +- keystone/server/backends.py | 2 +- keystone/tests/unit/backend/core_ldap.py | 2 +- .../tests/unit/common/test_connection_pool.py | 135 ---- keystone/tests/unit/core.py | 2 +- keystone/tests/unit/ksfixtures/cache.py | 6 +- keystone/tests/unit/test_backend_ldap.py | 2 +- keystone/tests/unit/test_cache.py | 324 -------- .../tests/unit/test_cache_backend_mongo.py | 728 ------------------ keystone/tests/unit/test_v3.py | 2 +- keystone/token/persistence/core.py | 6 +- keystone/token/provider.py | 2 +- requirements.txt | 1 + tox.ini | 18 +- 25 files changed, 76 insertions(+), 2145 deletions(-) delete mode 100644 keystone/tests/unit/common/test_connection_pool.py delete mode 100644 keystone/tests/unit/test_cache.py delete mode 100644 keystone/tests/unit/test_cache_backend_mongo.py diff --git a/config-generator/keystone.conf b/config-generator/keystone.conf index 0e00d489f5..fc5ef29cdb 100644 --- a/config-generator/keystone.conf +++ b/config-generator/keystone.conf @@ -3,6 +3,7 @@ output_file = etc/keystone.conf.sample wrap_width = 79 namespace = keystone namespace = keystone.notifications +namespace = oslo.cache namespace = oslo.log namespace = oslo.messaging namespace = oslo.policy diff --git a/keystone/assignment/core.py b/keystone/assignment/core.py index a510c3c194..d30f7c38f8 100644 --- a/keystone/assignment/core.py +++ b/keystone/assignment/core.py @@ -33,7 +33,7 @@ from keystone import notifications CONF = cfg.CONF LOG = log.getLogger(__name__) -MEMOIZE = cache.get_memoization_decorator(section='role') +MEMOIZE = cache.get_memoization_decorator(group='role') @dependency.provider('assignment_api') diff --git a/keystone/catalog/core.py b/keystone/catalog/core.py index 8bb7261917..5c005fa2a8 100644 --- a/keystone/catalog/core.py +++ b/keystone/catalog/core.py @@ -35,7 +35,7 @@ from keystone import notifications CONF = cfg.CONF LOG = log.getLogger(__name__) -MEMOIZE = cache.get_memoization_decorator(section='catalog') +MEMOIZE = cache.get_memoization_decorator(group='catalog') WHITELISTED_PROPERTIES = [ 'tenant_id', 'user_id', 'public_bind_host', 'admin_bind_host', 'compute_host', 'admin_port', 'public_port', diff --git a/keystone/common/cache/backends/memcache_pool.py b/keystone/common/cache/backends/memcache_pool.py index f3990b1264..b56a26007e 100644 --- a/keystone/common/cache/backends/memcache_pool.py +++ b/keystone/common/cache/backends/memcache_pool.py @@ -13,49 +13,16 @@ # License for the specific language governing permissions and limitations # under the License. -"""dogpile.cache backend that uses Memcached connection pool""" +"""This module is deprecated.""" -import functools -import logging - -from dogpile.cache.backends import memcached as memcached_backend - -from keystone.common.cache import _memcache_pool +from oslo_cache.backends import memcache_pool +from oslo_log import versionutils -LOG = logging.getLogger(__name__) - - -# Helper to ease backend refactoring -class ClientProxy(object): - def __init__(self, client_pool): - self.client_pool = client_pool - - def _run_method(self, __name, *args, **kwargs): - with self.client_pool.acquire() as client: - return getattr(client, __name)(*args, **kwargs) - - def __getattr__(self, name): - return functools.partial(self._run_method, name) - - -class PooledMemcachedBackend(memcached_backend.MemcachedBackend): - # Composed from GenericMemcachedBackend's and MemcacheArgs's __init__ - def __init__(self, arguments): - super(PooledMemcachedBackend, self).__init__(arguments) - self.client_pool = _memcache_pool.MemcacheClientPool( - self.url, - arguments={ - 'dead_retry': arguments.get('dead_retry', 5 * 60), - 'socket_timeout': arguments.get('socket_timeout', 3), - }, - maxsize=arguments.get('pool_maxsize', 10), - unused_timeout=arguments.get('pool_unused_timeout', 60), - conn_get_timeout=arguments.get('pool_connection_get_timeout', 10), - ) - - # Since all methods in backend just call one of methods of client, this - # lets us avoid need to hack it too much - @property - def client(self): - return ClientProxy(self.client_pool) +@versionutils.deprecated( + versionutils.deprecated.LIBERTY, + what='keystone.cache.memcache_pool backend', + in_favor_of='oslo_cache.memcache_pool backend', + remove_in=+1) +class PooledMemcachedBackend(memcache_pool.PooledMemcachedBackend): + pass diff --git a/keystone/common/cache/backends/mongo.py b/keystone/common/cache/backends/mongo.py index cb5ad8331a..4192eb87d1 100644 --- a/keystone/common/cache/backends/mongo.py +++ b/keystone/common/cache/backends/mongo.py @@ -12,550 +12,14 @@ # License for the specific language governing permissions and limitations # under the License. -import abc -import datetime +from oslo_cache.backends import mongo +from oslo_log import versionutils -from dogpile.cache import api -from dogpile.cache import util as dp_util -from oslo_log import log -from oslo_utils import importutils -from oslo_utils import timeutils -import six -from keystone import exception -from keystone.i18n import _, _LW - - -NO_VALUE = api.NO_VALUE -LOG = log.getLogger(__name__) - - -class MongoCacheBackend(api.CacheBackend): - """A MongoDB based caching backend implementing dogpile backend APIs. - - Arguments accepted in the arguments dictionary: - - :param db_hosts: string (required), hostname or IP address of the - MongoDB server instance. This can be a single MongoDB connection URI, - or a list of MongoDB connection URIs. - - :param db_name: string (required), the name of the database to be used. - - :param cache_collection: string (required), the name of collection to store - cached data. - *Note:* Different collection name can be provided if there is need to - create separate container (i.e. collection) for cache data. So region - configuration is done per collection. - - Following are optional parameters for MongoDB backend configuration, - - :param username: string, the name of the user to authenticate. - - :param password: string, the password of the user to authenticate. - - :param max_pool_size: integer, the maximum number of connections that the - pool will open simultaneously. By default the pool size is 10. - - :param w: integer, write acknowledgement for MongoDB client - - If not provided, then no default is set on MongoDB and then write - acknowledgement behavior occurs as per MongoDB default. This parameter - name is same as what is used in MongoDB docs. This value is specified - at collection level so its applicable to `cache_collection` db write - operations. - - If this is a replica set, write operations will block until they have - been replicated to the specified number or tagged set of servers. - Setting w=0 disables write acknowledgement and all other write concern - options. - - :param read_preference: string, the read preference mode for MongoDB client - Expected value is ``primary``, ``primaryPreferred``, ``secondary``, - ``secondaryPreferred``, or ``nearest``. This read_preference is - specified at collection level so its applicable to `cache_collection` - db read operations. - - :param use_replica: boolean, flag to indicate if replica client to be - used. Default is `False`. `replicaset_name` value is required if - `True`. - - :param replicaset_name: string, name of replica set. - Becomes required if `use_replica` is `True` - - :param son_manipulator: string, name of class with module name which - implements MongoDB SONManipulator. - Default manipulator used is :class:`.BaseTransform`. - - This manipulator is added per database. In multiple cache - configurations, the manipulator name should be same if same - database name ``db_name`` is used in those configurations. - - SONManipulator is used to manipulate custom data types as they are - saved or retrieved from MongoDB. Custom impl is only needed if cached - data is custom class and needs transformations when saving or reading - from db. If dogpile cached value contains built-in data types, then - BaseTransform class is sufficient as it already handles dogpile - CachedValue class transformation. - - :param mongo_ttl_seconds: integer, interval in seconds to indicate maximum - time-to-live value. - If value is greater than 0, then its assumed that cache_collection - needs to be TTL type (has index at 'doc_date' field). - By default, the value is -1 and its disabled. - Reference: - - .. NOTE:: - - This parameter is different from Dogpile own - expiration_time, which is the number of seconds after which Dogpile - will consider the value to be expired. When Dogpile considers a - value to be expired, it continues to use the value until generation - of a new value is complete, when using CacheRegion.get_or_create(). - Therefore, if you are setting `mongo_ttl_seconds`, you will want to - make sure it is greater than expiration_time by at least enough - seconds for new values to be generated, else the value would not - be available during a regeneration, forcing all threads to wait for - a regeneration each time a value expires. - - :param ssl: boolean, If True, create the connection to the server - using SSL. Default is `False`. Client SSL connection parameters depends - on server side SSL setup. For further reference on SSL configuration: - - - :param ssl_keyfile: string, the private keyfile used to identify the - local connection against mongod. If included with the certfile then - only the `ssl_certfile` is needed. Used only when `ssl` is `True`. - - :param ssl_certfile: string, the certificate file used to identify the - local connection against mongod. Used only when `ssl` is `True`. - - :param ssl_ca_certs: string, the ca_certs file contains a set of - concatenated 'certification authority' certificates, which are used to - validate certificates passed from the other end of the connection. - Used only when `ssl` is `True`. - - :param ssl_cert_reqs: string, the parameter cert_reqs specifies whether - a certificate is required from the other side of the connection, and - whether it will be validated if provided. It must be one of the three - values ``ssl.CERT_NONE`` (certificates ignored), ``ssl.CERT_OPTIONAL`` - (not required, but validated if provided), or - ``ssl.CERT_REQUIRED`` (required and validated). If the value of this - parameter is not ``ssl.CERT_NONE``, then the ssl_ca_certs parameter - must point to a file of CA certificates. Used only when `ssl` - is `True`. - - Rest of arguments are passed to mongo calls for read, write and remove. - So related options can be specified to pass to these operations. - - Further details of various supported arguments can be referred from - - - """ - - def __init__(self, arguments): - self.api = MongoApi(arguments) - - @dp_util.memoized_property - def client(self): - """Initializes MongoDB connection and collection defaults. - - This initialization is done only once and performed as part of lazy - inclusion of MongoDB dependency i.e. add imports only if related - backend is used. - - :return: :class:`.MongoApi` instance - """ - self.api.get_cache_collection() - return self.api - - def get(self, key): - value = self.client.get(key) - if value is None: - return NO_VALUE - else: - return value - - def get_multi(self, keys): - values = self.client.get_multi(keys) - return [ - NO_VALUE if key not in values - else values[key] for key in keys - ] - - def set(self, key, value): - self.client.set(key, value) - - def set_multi(self, mapping): - self.client.set_multi(mapping) - - def delete(self, key): - self.client.delete(key) - - def delete_multi(self, keys): - self.client.delete_multi(keys) - - -class MongoApi(object): - """Class handling MongoDB specific functionality. - - This class uses PyMongo APIs internally to create database connection - with configured pool size, ensures unique index on key, does database - authentication and ensure TTL collection index if configured so. - This class also serves as handle to cache collection for dogpile cache - APIs. - - In a single deployment, multiple cache configuration can be defined. In - that case of multiple cache collections usage, db client connection pool - is shared when cache collections are within same database. - """ - - # class level attributes for re-use of db client connection and collection - _DB = {} # dict of db_name: db connection reference - _MONGO_COLLS = {} # dict of cache_collection : db collection reference - - def __init__(self, arguments): - self._init_args(arguments) - self._data_manipulator = None - - def _init_args(self, arguments): - """Helper logic for collecting and parsing MongoDB specific arguments. - - The arguments passed in are separated out in connection specific - setting and rest of arguments are passed to create/update/delete - db operations. - """ - self.conn_kwargs = {} # connection specific arguments - - self.hosts = arguments.pop('db_hosts', None) - if self.hosts is None: - msg = _('db_hosts value is required') - raise exception.ValidationError(message=msg) - - self.db_name = arguments.pop('db_name', None) - if self.db_name is None: - msg = _('database db_name is required') - raise exception.ValidationError(message=msg) - - self.cache_collection = arguments.pop('cache_collection', None) - if self.cache_collection is None: - msg = _('cache_collection name is required') - raise exception.ValidationError(message=msg) - - self.username = arguments.pop('username', None) - self.password = arguments.pop('password', None) - self.max_pool_size = arguments.pop('max_pool_size', 10) - - self.w = arguments.pop('w', -1) - try: - self.w = int(self.w) - except ValueError: - msg = _('integer value expected for w (write concern attribute)') - raise exception.ValidationError(message=msg) - - self.read_preference = arguments.pop('read_preference', None) - - self.use_replica = arguments.pop('use_replica', False) - if self.use_replica: - if arguments.get('replicaset_name') is None: - msg = _('replicaset_name required when use_replica is True') - raise exception.ValidationError(message=msg) - self.replicaset_name = arguments.get('replicaset_name') - - self.son_manipulator = arguments.pop('son_manipulator', None) - - # set if mongo collection needs to be TTL type. - # This needs to be max ttl for any cache entry. - # By default, -1 means don't use TTL collection. - # With ttl set, it creates related index and have doc_date field with - # needed expiration interval - self.ttl_seconds = arguments.pop('mongo_ttl_seconds', -1) - try: - self.ttl_seconds = int(self.ttl_seconds) - except ValueError: - msg = _('integer value expected for mongo_ttl_seconds') - raise exception.ValidationError(message=msg) - - self.conn_kwargs['ssl'] = arguments.pop('ssl', False) - if self.conn_kwargs['ssl']: - ssl_keyfile = arguments.pop('ssl_keyfile', None) - ssl_certfile = arguments.pop('ssl_certfile', None) - ssl_ca_certs = arguments.pop('ssl_ca_certs', None) - ssl_cert_reqs = arguments.pop('ssl_cert_reqs', None) - if ssl_keyfile: - self.conn_kwargs['ssl_keyfile'] = ssl_keyfile - if ssl_certfile: - self.conn_kwargs['ssl_certfile'] = ssl_certfile - if ssl_ca_certs: - self.conn_kwargs['ssl_ca_certs'] = ssl_ca_certs - if ssl_cert_reqs: - self.conn_kwargs['ssl_cert_reqs'] = ( - self._ssl_cert_req_type(ssl_cert_reqs)) - - # rest of arguments are passed to mongo crud calls - self.meth_kwargs = arguments - - def _ssl_cert_req_type(self, req_type): - try: - import ssl - except ImportError: - raise exception.ValidationError(_('no ssl support available')) - req_type = req_type.upper() - try: - return { - 'NONE': ssl.CERT_NONE, - 'OPTIONAL': ssl.CERT_OPTIONAL, - 'REQUIRED': ssl.CERT_REQUIRED - }[req_type] - except KeyError: - msg = _('Invalid ssl_cert_reqs value of %s, must be one of ' - '"NONE", "OPTIONAL", "REQUIRED"') % (req_type) - raise exception.ValidationError(message=msg) - - def _get_db(self): - # defer imports until backend is used - global pymongo - import pymongo - if self.use_replica: - connection = pymongo.MongoReplicaSetClient( - host=self.hosts, replicaSet=self.replicaset_name, - max_pool_size=self.max_pool_size, **self.conn_kwargs) - else: # used for standalone node or mongos in sharded setup - connection = pymongo.MongoClient( - host=self.hosts, max_pool_size=self.max_pool_size, - **self.conn_kwargs) - - database = getattr(connection, self.db_name) - - self._assign_data_mainpulator() - database.add_son_manipulator(self._data_manipulator) - if self.username and self.password: - database.authenticate(self.username, self.password) - return database - - def _assign_data_mainpulator(self): - if self._data_manipulator is None: - if self.son_manipulator: - self._data_manipulator = importutils.import_object( - self.son_manipulator) - else: - self._data_manipulator = BaseTransform() - - def _get_doc_date(self): - if self.ttl_seconds > 0: - expire_delta = datetime.timedelta(seconds=self.ttl_seconds) - doc_date = timeutils.utcnow() + expire_delta - else: - doc_date = timeutils.utcnow() - return doc_date - - def get_cache_collection(self): - if self.cache_collection not in self._MONGO_COLLS: - global pymongo - import pymongo - # re-use db client connection if already defined as part of - # earlier dogpile cache configuration - if self.db_name not in self._DB: - self._DB[self.db_name] = self._get_db() - coll = getattr(self._DB[self.db_name], self.cache_collection) - - self._assign_data_mainpulator() - if self.read_preference: - # pymongo 3.0 renamed mongos_enum to read_pref_mode_from_name - f = getattr(pymongo.read_preferences, - 'read_pref_mode_from_name', None) - if not f: - f = pymongo.read_preferences.mongos_enum - self.read_preference = f(self.read_preference) - coll.read_preference = self.read_preference - if self.w > -1: - coll.write_concern['w'] = self.w - if self.ttl_seconds > 0: - kwargs = {'expireAfterSeconds': self.ttl_seconds} - coll.ensure_index('doc_date', cache_for=5, **kwargs) - else: - self._validate_ttl_index(coll, self.cache_collection, - self.ttl_seconds) - self._MONGO_COLLS[self.cache_collection] = coll - - return self._MONGO_COLLS[self.cache_collection] - - def _get_cache_entry(self, key, value, meta, doc_date): - """MongoDB cache data representation. - - Storing cache key as ``_id`` field as MongoDB by default creates - unique index on this field. So no need to create separate field and - index for storing cache key. Cache data has additional ``doc_date`` - field for MongoDB TTL collection support. - """ - return dict(_id=key, value=value, meta=meta, doc_date=doc_date) - - def _validate_ttl_index(self, collection, coll_name, ttl_seconds): - """Checks if existing TTL index is removed on a collection. - - This logs warning when existing collection has TTL index defined and - new cache configuration tries to disable index with - ``mongo_ttl_seconds < 0``. In that case, existing index needs - to be addressed first to make new configuration effective. - Refer to MongoDB documentation around TTL index for further details. - """ - indexes = collection.index_information() - for indx_name, index_data in indexes.items(): - if all(k in index_data for k in ('key', 'expireAfterSeconds')): - existing_value = index_data['expireAfterSeconds'] - fld_present = 'doc_date' in index_data['key'][0] - if fld_present and existing_value > -1 and ttl_seconds < 1: - msg = _LW('TTL index already exists on db collection ' - '<%(c_name)s>, remove index <%(indx_name)s> ' - 'first to make updated mongo_ttl_seconds value ' - 'to be effective') - LOG.warn(msg, {'c_name': coll_name, - 'indx_name': indx_name}) - - def get(self, key): - critieria = {'_id': key} - result = self.get_cache_collection().find_one(spec_or_id=critieria, - **self.meth_kwargs) - if result: - return result['value'] - else: - return None - - def get_multi(self, keys): - db_results = self._get_results_as_dict(keys) - return {doc['_id']: doc['value'] for doc in six.itervalues(db_results)} - - def _get_results_as_dict(self, keys): - critieria = {'_id': {'$in': keys}} - db_results = self.get_cache_collection().find(spec=critieria, - **self.meth_kwargs) - return {doc['_id']: doc for doc in db_results} - - def set(self, key, value): - doc_date = self._get_doc_date() - ref = self._get_cache_entry(key, value.payload, value.metadata, - doc_date) - spec = {'_id': key} - # find and modify does not have manipulator support - # so need to do conversion as part of input document - ref = self._data_manipulator.transform_incoming(ref, self) - self.get_cache_collection().find_and_modify(spec, ref, upsert=True, - **self.meth_kwargs) - - def set_multi(self, mapping): - """Insert multiple documents specified as key, value pairs. - - In this case, multiple documents can be added via insert provided they - do not exist. - Update of multiple existing documents is done one by one - """ - doc_date = self._get_doc_date() - insert_refs = [] - update_refs = [] - existing_docs = self._get_results_as_dict(list(mapping.keys())) - for key, value in mapping.items(): - ref = self._get_cache_entry(key, value.payload, value.metadata, - doc_date) - if key in existing_docs: - ref['_id'] = existing_docs[key]['_id'] - update_refs.append(ref) - else: - insert_refs.append(ref) - if insert_refs: - self.get_cache_collection().insert(insert_refs, manipulate=True, - **self.meth_kwargs) - for upd_doc in update_refs: - self.get_cache_collection().save(upd_doc, manipulate=True, - **self.meth_kwargs) - - def delete(self, key): - critieria = {'_id': key} - self.get_cache_collection().remove(spec_or_id=critieria, - **self.meth_kwargs) - - def delete_multi(self, keys): - critieria = {'_id': {'$in': keys}} - self.get_cache_collection().remove(spec_or_id=critieria, - **self.meth_kwargs) - - -@six.add_metaclass(abc.ABCMeta) -class AbstractManipulator(object): - """Abstract class with methods which need to be implemented for custom - manipulation. - - Adding this as a base class for :class:`.BaseTransform` instead of adding - import dependency of pymongo specific class i.e. - `pymongo.son_manipulator.SONManipulator` and using that as base class. - This is done to avoid pymongo dependency if MongoDB backend is not used. - """ - @abc.abstractmethod - def transform_incoming(self, son, collection): - """Used while saving data to MongoDB. - - :param son: the SON object to be inserted into the database - :param collection: the collection the object is being inserted into - - :returns: transformed SON object - - """ - raise exception.NotImplemented() # pragma: no cover - - @abc.abstractmethod - def transform_outgoing(self, son, collection): - """Used while reading data from MongoDB. - - :param son: the SON object being retrieved from the database - :param collection: the collection this object was stored in - - :returns: transformed SON object - """ - raise exception.NotImplemented() # pragma: no cover - - def will_copy(self): - """Will this SON manipulator make a copy of the incoming document? - - Derived classes that do need to make a copy should override this - method, returning `True` instead of `False`. - - :returns: boolean - """ - return False - - -class BaseTransform(AbstractManipulator): - """Base transformation class to store and read dogpile cached data - from MongoDB. - - This is needed as dogpile internally stores data as a custom class - i.e. dogpile.cache.api.CachedValue - - Note: Custom manipulator needs to always override ``transform_incoming`` - and ``transform_outgoing`` methods. MongoDB manipulator logic specifically - checks that overridden method in instance and its super are different. - """ - - def transform_incoming(self, son, collection): - """Used while saving data to MongoDB.""" - for (key, value) in list(son.items()): - if isinstance(value, api.CachedValue): - son[key] = value.payload # key is 'value' field here - son['meta'] = value.metadata - elif isinstance(value, dict): # Make sure we recurse into sub-docs - son[key] = self.transform_incoming(value, collection) - return son - - def transform_outgoing(self, son, collection): - """Used while reading data from MongoDB.""" - metadata = None - # make sure its top level dictionary with all expected fields names - # present - if isinstance(son, dict) and all(k in son for k in - ('_id', 'value', 'meta', 'doc_date')): - payload = son.pop('value', None) - metadata = son.pop('meta', None) - for (key, value) in list(son.items()): - if isinstance(value, dict): - son[key] = self.transform_outgoing(value, collection) - if metadata is not None: - son['value'] = api.CachedValue(payload, metadata) - return son +@versionutils.deprecated( + versionutils.deprecated.LIBERTY, + what='keystone.cache.mongo backend', + in_favor_of='oslo_cache.mongo backend', + remove_in=+1) +class MongoCacheBackend(mongo.MongoCacheBackend): + pass diff --git a/keystone/common/cache/backends/noop.py b/keystone/common/cache/backends/noop.py index 38329c9401..05859f8859 100644 --- a/keystone/common/cache/backends/noop.py +++ b/keystone/common/cache/backends/noop.py @@ -13,11 +13,17 @@ # under the License. from dogpile.cache import api +from oslo_log import versionutils NO_VALUE = api.NO_VALUE +@versionutils.deprecated( + versionutils.deprecated.LIBERTY, + what='keystone.common.cache.noop backend', + in_favor_of="dogpile.cache's Null backend", + remove_in=+1) class NoopCacheBackend(api.CacheBackend): """A no op backend as a default caching backend. diff --git a/keystone/common/cache/core.py b/keystone/common/cache/core.py index 306587b317..e8db82d5c2 100644 --- a/keystone/common/cache/core.py +++ b/keystone/common/cache/core.py @@ -15,21 +15,25 @@ """Keystone Caching Layer Implementation.""" import dogpile.cache -from dogpile.cache import proxy -from dogpile.cache import util +from oslo_cache import core as cache from oslo_config import cfg -from oslo_log import log -from oslo_utils import importutils - -from keystone import exception -from keystone.i18n import _, _LE CONF = cfg.CONF -LOG = log.getLogger(__name__) +CACHE_REGION = cache.create_region() -make_region = dogpile.cache.make_region +def configure_cache(): + cache.configure_cache_region(CONF, CACHE_REGION) + + +def get_memoization_decorator(group, expiration_group=None): + return cache.get_memoization_decorator(CONF, CACHE_REGION, group, + expiration_group=expiration_group) + + +# NOTE(stevemar): When memcache_pool, mongo and noop backends are removed +# we no longer need to register the backends here. dogpile.cache.register_backend( 'keystone.common.cache.noop', 'keystone.common.cache.backends.noop', @@ -44,265 +48,3 @@ dogpile.cache.register_backend( 'keystone.cache.memcache_pool', 'keystone.common.cache.backends.memcache_pool', 'PooledMemcachedBackend') - - -class DebugProxy(proxy.ProxyBackend): - """Extra Logging ProxyBackend.""" - # NOTE(morganfainberg): Pass all key/values through repr to ensure we have - # a clean description of the information. Without use of repr, it might - # be possible to run into encode/decode error(s). For logging/debugging - # purposes encode/decode is irrelevant and we should be looking at the - # data exactly as it stands. - - def get(self, key): - value = self.proxied.get(key) - LOG.debug('CACHE_GET: Key: "%(key)r" Value: "%(value)r"', - {'key': key, 'value': value}) - return value - - def get_multi(self, keys): - values = self.proxied.get_multi(keys) - LOG.debug('CACHE_GET_MULTI: "%(keys)r" Values: "%(values)r"', - {'keys': keys, 'values': values}) - return values - - def set(self, key, value): - LOG.debug('CACHE_SET: Key: "%(key)r" Value: "%(value)r"', - {'key': key, 'value': value}) - return self.proxied.set(key, value) - - def set_multi(self, keys): - LOG.debug('CACHE_SET_MULTI: "%r"', keys) - self.proxied.set_multi(keys) - - def delete(self, key): - self.proxied.delete(key) - LOG.debug('CACHE_DELETE: "%r"', key) - - def delete_multi(self, keys): - LOG.debug('CACHE_DELETE_MULTI: "%r"', keys) - self.proxied.delete_multi(keys) - - -def build_cache_config(): - """Build the cache region dictionary configuration. - - :returns: dict - """ - prefix = CONF.cache.config_prefix - conf_dict = {} - conf_dict['%s.backend' % prefix] = CONF.cache.backend - conf_dict['%s.expiration_time' % prefix] = CONF.cache.expiration_time - for argument in CONF.cache.backend_argument: - try: - (argname, argvalue) = argument.split(':', 1) - except ValueError: - msg = _LE('Unable to build cache config-key. Expected format ' - '":". Skipping unknown format: %s') - LOG.error(msg, argument) - continue - - arg_key = '.'.join([prefix, 'arguments', argname]) - conf_dict[arg_key] = argvalue - - LOG.debug('Keystone Cache Config: %s', conf_dict) - # NOTE(yorik-sar): these arguments will be used for memcache-related - # backends. Use setdefault for url to support old-style setting through - # backend_argument=url:127.0.0.1:11211 - conf_dict.setdefault('%s.arguments.url' % prefix, - CONF.cache.memcache_servers) - for arg in ('dead_retry', 'socket_timeout', 'pool_maxsize', - 'pool_unused_timeout', 'pool_connection_get_timeout'): - value = getattr(CONF.cache, 'memcache_' + arg) - conf_dict['%s.arguments.%s' % (prefix, arg)] = value - - return conf_dict - - -def configure_cache_region(region): - """Configure a cache region. - - :param region: optional CacheRegion object, if not provided a new region - will be instantiated - :raises: exception.ValidationError - :returns: dogpile.cache.CacheRegion - """ - if not isinstance(region, dogpile.cache.CacheRegion): - raise exception.ValidationError( - _('region not type dogpile.cache.CacheRegion')) - - if not region.is_configured: - # NOTE(morganfainberg): this is how you tell if a region is configured. - # There is a request logged with dogpile.cache upstream to make this - # easier / less ugly. - - config_dict = build_cache_config() - region.configure_from_config(config_dict, - '%s.' % CONF.cache.config_prefix) - - if CONF.cache.debug_cache_backend: - region.wrap(DebugProxy) - - # NOTE(morganfainberg): if the backend requests the use of a - # key_mangler, we should respect that key_mangler function. If a - # key_mangler is not defined by the backend, use the sha1_mangle_key - # mangler provided by dogpile.cache. This ensures we always use a fixed - # size cache-key. - if region.key_mangler is None: - region.key_mangler = util.sha1_mangle_key - - for class_path in CONF.cache.proxies: - # NOTE(morganfainberg): if we have any proxy wrappers, we should - # ensure they are added to the cache region's backend. Since - # configure_from_config doesn't handle the wrap argument, we need - # to manually add the Proxies. For information on how the - # ProxyBackends work, see the dogpile.cache documents on - # "changing-backend-behavior" - cls = importutils.import_class(class_path) - LOG.debug("Adding cache-proxy '%s' to backend.", class_path) - region.wrap(cls) - - return region - - -def get_should_cache_fn(section): - """Build a function that returns a config section's caching status. - - For any given driver in keystone that has caching capabilities, a boolean - config option for that driver's section (e.g. ``token``) should exist and - default to ``True``. This function will use that value to tell the caching - decorator if caching for that driver is enabled. To properly use this - with the decorator, pass this function the configuration section and assign - the result to a variable. Pass the new variable to the caching decorator - as the named argument ``should_cache_fn``. e.g.:: - - from keystone.common import cache - - SHOULD_CACHE = cache.get_should_cache_fn('token') - - @cache.on_arguments(should_cache_fn=SHOULD_CACHE) - def function(arg1, arg2): - ... - - :param section: name of the configuration section to examine - :type section: string - :returns: function reference - """ - def should_cache(value): - if not CONF.cache.enabled: - return False - conf_group = getattr(CONF, section) - return getattr(conf_group, 'caching', True) - return should_cache - - -def get_expiration_time_fn(section): - """Build a function that returns a config section's expiration time status. - - For any given driver in keystone that has caching capabilities, an int - config option called ``cache_time`` for that driver's section - (e.g. ``token``) should exist and typically default to ``None``. This - function will use that value to tell the caching decorator of the TTL - override for caching the resulting objects. If the value of the config - option is ``None`` the default value provided in the - ``[cache] expiration_time`` option will be used by the decorator. The - default may be set to something other than ``None`` in cases where the - caching TTL should not be tied to the global default(s) (e.g. - revocation_list changes very infrequently and can be cached for >1h by - default). - - To properly use this with the decorator, pass this function the - configuration section and assign the result to a variable. Pass the new - variable to the caching decorator as the named argument - ``expiration_time``. e.g.:: - - from keystone.common import cache - - EXPIRATION_TIME = cache.get_expiration_time_fn('token') - - @cache.on_arguments(expiration_time=EXPIRATION_TIME) - def function(arg1, arg2): - ... - - :param section: name of the configuration section to examine - :type section: string - :rtype: function reference - """ - def get_expiration_time(): - conf_group = getattr(CONF, section) - return getattr(conf_group, 'cache_time', None) - return get_expiration_time - - -def key_generate_to_str(s): - # NOTE(morganfainberg): Since we need to stringify all arguments, attempt - # to stringify and handle the Unicode error explicitly as needed. - try: - return str(s) - except UnicodeEncodeError: - return s.encode('utf-8') - - -def function_key_generator(namespace, fn, to_str=key_generate_to_str): - # NOTE(morganfainberg): This wraps dogpile.cache's default - # function_key_generator to change the default to_str mechanism. - return util.function_key_generator(namespace, fn, to_str=to_str) - - -REGION = dogpile.cache.make_region( - function_key_generator=function_key_generator) -on_arguments = REGION.cache_on_arguments - - -def get_memoization_decorator(section, expiration_section=None): - """Build a function based on the `on_arguments` decorator for the section. - - For any given driver in Keystone that has caching capabilities, a - pair of functions is required to properly determine the status of the - caching capabilities (a toggle to indicate caching is enabled and any - override of the default TTL for cached data). This function will return - an object that has the memoization decorator ``on_arguments`` - pre-configured for the driver. - - Example usage:: - - from keystone.common import cache - - MEMOIZE = cache.get_memoization_decorator(section='token') - - @MEMOIZE - def function(arg1, arg2): - ... - - - ALTERNATE_MEMOIZE = cache.get_memoization_decorator( - section='token', expiration_section='revoke') - - @ALTERNATE_MEMOIZE - def function2(arg1, arg2): - ... - - :param section: name of the configuration section to examine - :type section: string - :param expiration_section: name of the configuration section to examine - for the expiration option. This will fall back - to using ``section`` if the value is unspecified - or ``None`` - :type expiration_section: string - :rtype: function reference - """ - if expiration_section is None: - expiration_section = section - should_cache = get_should_cache_fn(section) - expiration_time = get_expiration_time_fn(expiration_section) - - memoize = REGION.cache_on_arguments(should_cache_fn=should_cache, - expiration_time=expiration_time) - - # Make sure the actual "should_cache" and "expiration_time" methods are - # available. This is potentially interesting/useful to pre-seed cache - # values. - memoize.should_cache = should_cache - memoize.get_expiration_time = expiration_time - - return memoize diff --git a/keystone/common/config.py b/keystone/common/config.py index f285d3866f..c480c9e075 100644 --- a/keystone/common/config.py +++ b/keystone/common/config.py @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo_cache import core as cache from oslo_config import cfg import oslo_messaging import passlib.utils @@ -306,82 +307,6 @@ FILE_OPTIONS = { deprecated_opts=[cfg.DeprecatedOpt( 'revocation_cache_time', group='token')]), ], - 'cache': [ - cfg.StrOpt('config_prefix', default='cache.keystone', - help='Prefix for building the configuration dictionary ' - 'for the cache region. This should not need to be ' - 'changed unless there is another dogpile.cache ' - 'region with the same configuration name.'), - cfg.IntOpt('expiration_time', default=600, - help='Default TTL, in seconds, for any cached item in ' - 'the dogpile.cache region. This applies to any ' - 'cached method that doesn\'t have an explicit ' - 'cache expiration time defined for it.'), - # NOTE(morganfainberg): the dogpile.cache.memory acceptable in devstack - # and other such single-process/thread deployments. Running - # dogpile.cache.memory in any other configuration has the same pitfalls - # as the KVS token backend. It is recommended that either Redis or - # Memcached are used as the dogpile backend for real workloads. To - # prevent issues with the memory cache ending up in "production" - # unintentionally, we register a no-op as the keystone default caching - # backend. - cfg.StrOpt('backend', default='keystone.common.cache.noop', - help='Dogpile.cache backend module. It is recommended ' - 'that Memcache with pooling ' - '(keystone.cache.memcache_pool) or Redis ' - '(dogpile.cache.redis) be used in production ' - 'deployments. Small workloads (single process) ' - 'like devstack can use the dogpile.cache.memory ' - 'backend.'), - cfg.MultiStrOpt('backend_argument', default=[], secret=True, - help='Arguments supplied to the backend module. ' - 'Specify this option once per argument to be ' - 'passed to the dogpile.cache backend. Example ' - 'format: ":".'), - cfg.ListOpt('proxies', default=[], - help='Proxy classes to import that will affect the way ' - 'the dogpile.cache backend functions. See the ' - 'dogpile.cache documentation on ' - 'changing-backend-behavior.'), - cfg.BoolOpt('enabled', default=False, - help='Global toggle for all caching using the ' - 'should_cache_fn mechanism.'), - cfg.BoolOpt('debug_cache_backend', default=False, - help='Extra debugging from the cache backend (cache ' - 'keys, get/set/delete/etc calls). This is only ' - 'really useful if you need to see the specific ' - 'cache-backend get/set/delete calls with the ' - 'keys/values. Typically this should be left set ' - 'to false.'), - cfg.ListOpt('memcache_servers', default=['localhost:11211'], - help='Memcache servers in the format of "host:port".' - ' (dogpile.cache.memcache and keystone.cache.memcache_pool' - ' backends only).'), - cfg.IntOpt('memcache_dead_retry', - default=5 * 60, - help='Number of seconds memcached server is considered dead' - ' before it is tried again. (dogpile.cache.memcache and' - ' keystone.cache.memcache_pool backends only).'), - cfg.IntOpt('memcache_socket_timeout', - default=3, - help='Timeout in seconds for every call to a server.' - ' (dogpile.cache.memcache and keystone.cache.memcache_pool' - ' backends only).'), - cfg.IntOpt('memcache_pool_maxsize', - default=10, - help='Max total number of open connections to every' - ' memcached server. (keystone.cache.memcache_pool backend' - ' only).'), - cfg.IntOpt('memcache_pool_unused_timeout', - default=60, - help='Number of seconds a connection to memcached is held' - ' unused in the pool before it is closed.' - ' (keystone.cache.memcache_pool backend only).'), - cfg.IntOpt('memcache_pool_connection_get_timeout', - default=10, - help='Number of seconds that an operation will wait to get ' - 'a memcache client connection.'), - ], 'ssl': [ cfg.StrOpt('ca_key', default='/etc/keystone/ssl/private/cakey.pem', @@ -1212,6 +1137,8 @@ def configure(conf=None): # register any non-default auth methods here (used by extensions, etc) setup_authentication(conf) + # add oslo.cache related config options + cache.configure(conf) def list_opts(): diff --git a/keystone/common/kvs/backends/memcached.py b/keystone/common/kvs/backends/memcached.py index 8e8928760d..a9e8833cea 100644 --- a/keystone/common/kvs/backends/memcached.py +++ b/keystone/common/kvs/backends/memcached.py @@ -21,11 +21,11 @@ import time from dogpile.cache import api from dogpile.cache.backends import memcached +from oslo_cache.backends import memcache_pool from oslo_config import cfg from oslo_log import log from six.moves import range -from keystone.common.cache.backends import memcache_pool from keystone import exception from keystone.i18n import _ diff --git a/keystone/contrib/revoke/core.py b/keystone/contrib/revoke/core.py index 3b108c9ece..5dd6b8a33e 100644 --- a/keystone/contrib/revoke/core.py +++ b/keystone/contrib/revoke/core.py @@ -54,7 +54,7 @@ EXTENSION_DATA = { extension.register_admin_extension(EXTENSION_DATA['alias'], EXTENSION_DATA) extension.register_public_extension(EXTENSION_DATA['alias'], EXTENSION_DATA) -MEMOIZE = cache.get_memoization_decorator(section='revoke') +MEMOIZE = cache.get_memoization_decorator(group='revoke') def revoked_before_cutoff_time(): diff --git a/keystone/identity/core.py b/keystone/identity/core.py index e3b70eb485..e64a82a18a 100644 --- a/keystone/identity/core.py +++ b/keystone/identity/core.py @@ -39,7 +39,7 @@ CONF = cfg.CONF LOG = log.getLogger(__name__) -MEMOIZE = cache.get_memoization_decorator(section='identity') +MEMOIZE = cache.get_memoization_decorator(group='identity') DOMAIN_CONF_FHEAD = 'keystone.' DOMAIN_CONF_FTAIL = '.conf' diff --git a/keystone/resource/core.py b/keystone/resource/core.py index 6015107d2a..d6383ac52f 100644 --- a/keystone/resource/core.py +++ b/keystone/resource/core.py @@ -30,7 +30,7 @@ from keystone import notifications CONF = cfg.CONF LOG = log.getLogger(__name__) -MEMOIZE = cache.get_memoization_decorator(section='resource') +MEMOIZE = cache.get_memoization_decorator(group='resource') def calc_default_domain(): @@ -810,7 +810,7 @@ class ResourceDriverV8(object): Driver = manager.create_legacy_driver(ResourceDriverV8) -MEMOIZE_CONFIG = cache.get_memoization_decorator(section='domain_config') +MEMOIZE_CONFIG = cache.get_memoization_decorator(group='domain_config') @dependency.provider('domain_config_api') diff --git a/keystone/server/backends.py b/keystone/server/backends.py index ebe00a8177..dfc6b6565f 100644 --- a/keystone/server/backends.py +++ b/keystone/server/backends.py @@ -30,7 +30,7 @@ from keystone import trust def load_backends(): # Configure and build the cache - cache.configure_cache_region(cache.REGION) + cache.configure_cache() # Ensure that the identity driver is created before the assignment manager # and that the assignment driver is created before the resource manager. diff --git a/keystone/tests/unit/backend/core_ldap.py b/keystone/tests/unit/backend/core_ldap.py index 869bb620ac..fa36b92683 100644 --- a/keystone/tests/unit/backend/core_ldap.py +++ b/keystone/tests/unit/backend/core_ldap.py @@ -113,7 +113,7 @@ class BaseBackendLdapIdentitySqlEverythingElse(unit.SQLDriverOverrides): super(BaseBackendLdapIdentitySqlEverythingElse, self).setUp() self.clear_database() self.load_backends() - cache.configure_cache_region(cache.REGION) + cache.configure_cache() sqldb.recreate() self.load_fixtures(default_fixtures) diff --git a/keystone/tests/unit/common/test_connection_pool.py b/keystone/tests/unit/common/test_connection_pool.py deleted file mode 100644 index 3813e03395..0000000000 --- a/keystone/tests/unit/common/test_connection_pool.py +++ /dev/null @@ -1,135 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import threading -import time - -import mock -import six -from six.moves import queue -import testtools -from testtools import matchers - -from keystone.common.cache import _memcache_pool -from keystone import exception -from keystone.tests.unit import core - - -class _TestConnectionPool(_memcache_pool.ConnectionPool): - destroyed_value = 'destroyed' - - def _create_connection(self): - return mock.MagicMock() - - def _destroy_connection(self, conn): - conn(self.destroyed_value) - - -class TestConnectionPool(core.TestCase): - def setUp(self): - super(TestConnectionPool, self).setUp() - self.unused_timeout = 10 - self.maxsize = 2 - self.connection_pool = _TestConnectionPool( - maxsize=self.maxsize, - unused_timeout=self.unused_timeout) - self.addCleanup(self.cleanup_instance('connection_pool')) - - def test_get_context_manager(self): - self.assertThat(self.connection_pool.queue, matchers.HasLength(0)) - with self.connection_pool.acquire() as conn: - self.assertEqual(1, self.connection_pool._acquired) - self.assertEqual(0, self.connection_pool._acquired) - self.assertThat(self.connection_pool.queue, matchers.HasLength(1)) - self.assertEqual(conn, self.connection_pool.queue[0].connection) - - def test_cleanup_pool(self): - self.test_get_context_manager() - newtime = time.time() + self.unused_timeout * 2 - non_expired_connection = _memcache_pool._PoolItem( - ttl=(newtime * 2), - connection=mock.MagicMock()) - self.connection_pool.queue.append(non_expired_connection) - self.assertThat(self.connection_pool.queue, matchers.HasLength(2)) - with mock.patch.object(time, 'time', return_value=newtime): - conn = self.connection_pool.queue[0].connection - with self.connection_pool.acquire(): - pass - conn.assert_has_calls( - [mock.call(self.connection_pool.destroyed_value)]) - self.assertThat(self.connection_pool.queue, matchers.HasLength(1)) - self.assertEqual(0, non_expired_connection.connection.call_count) - - def test_acquire_conn_exception_returns_acquired_count(self): - class TestException(Exception): - pass - - with mock.patch.object(_TestConnectionPool, '_create_connection', - side_effect=TestException): - with testtools.ExpectedException(TestException): - with self.connection_pool.acquire(): - pass - self.assertThat(self.connection_pool.queue, - matchers.HasLength(0)) - self.assertEqual(0, self.connection_pool._acquired) - - def test_connection_pool_limits_maximum_connections(self): - # NOTE(morganfainberg): To ensure we don't lockup tests until the - # job limit, explicitly call .get_nowait() and .put_nowait() in this - # case. - conn1 = self.connection_pool.get_nowait() - conn2 = self.connection_pool.get_nowait() - - # Use a nowait version to raise an Empty exception indicating we would - # not get another connection until one is placed back into the queue. - self.assertRaises(queue.Empty, self.connection_pool.get_nowait) - - # Place the connections back into the pool. - self.connection_pool.put_nowait(conn1) - self.connection_pool.put_nowait(conn2) - - # Make sure we can get a connection out of the pool again. - self.connection_pool.get_nowait() - - def test_connection_pool_maximum_connection_get_timeout(self): - connection_pool = _TestConnectionPool( - maxsize=1, - unused_timeout=self.unused_timeout, - conn_get_timeout=0) - - def _acquire_connection(): - with connection_pool.acquire(): - pass - - # Make sure we've consumed the only available connection from the pool - conn = connection_pool.get_nowait() - - self.assertRaises(exception.UnexpectedError, _acquire_connection) - - # Put the connection back and ensure we can acquire the connection - # after it is available. - connection_pool.put_nowait(conn) - _acquire_connection() - - -class TestMemcacheClientOverrides(core.BaseTestCase): - - def test_client_stripped_of_threading_local(self): - """threading.local overrides are restored for _MemcacheClient""" - client_class = _memcache_pool._MemcacheClient - # get the genuine thread._local from MRO - thread_local = client_class.__mro__[2] - self.assertTrue(thread_local is threading.local) - for field in six.iterkeys(thread_local.__dict__): - if field not in ('__dict__', '__weakref__'): - self.assertNotEqual(id(getattr(thread_local, field, None)), - id(getattr(client_class, field, None))) diff --git a/keystone/tests/unit/core.py b/keystone/tests/unit/core.py index eb8b9f65b8..1b2140c224 100644 --- a/keystone/tests/unit/core.py +++ b/keystone/tests/unit/core.py @@ -411,7 +411,7 @@ class TestCase(BaseTestCase): group='cache', backend='dogpile.cache.memory', enabled=True, - proxies=['keystone.tests.unit.test_cache.CacheIsolatingProxy']) + proxies=['oslo_cache.testing.CacheIsolatingProxy']) self.config_fixture.config( group='catalog', driver='templated', diff --git a/keystone/tests/unit/ksfixtures/cache.py b/keystone/tests/unit/ksfixtures/cache.py index 74566f1ebc..1160a1f4a9 100644 --- a/keystone/tests/unit/ksfixtures/cache.py +++ b/keystone/tests/unit/ksfixtures/cache.py @@ -29,8 +29,8 @@ class Cache(fixtures.Fixture): # NOTE(morganfainberg): The only way to reconfigure the CacheRegion # object on each setUp() call is to remove the .backend property. - if cache.REGION.is_configured: - del cache.REGION.backend + if cache.CACHE_REGION.is_configured: + del cache.CACHE_REGION.backend # ensure the cache region instance is setup - cache.configure_cache_region(cache.REGION) + cache.configure_cache() diff --git a/keystone/tests/unit/test_backend_ldap.py b/keystone/tests/unit/test_backend_ldap.py index 8874b34253..2a9f77e7d2 100644 --- a/keystone/tests/unit/test_backend_ldap.py +++ b/keystone/tests/unit/test_backend_ldap.py @@ -2311,7 +2311,7 @@ class LdapIdentitySqlAssignment(BaseLDAPIdentity, unit.SQLDriverOverrides, super(LdapIdentitySqlAssignment, self).setUp() self.ldapdb.clear() self.load_backends() - cache.configure_cache_region(cache.REGION) + cache.configure_cache() sqldb.recreate() self.load_fixtures(default_fixtures) diff --git a/keystone/tests/unit/test_cache.py b/keystone/tests/unit/test_cache.py deleted file mode 100644 index 3c2afe6687..0000000000 --- a/keystone/tests/unit/test_cache.py +++ /dev/null @@ -1,324 +0,0 @@ -# Copyright 2013 Metacloud -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import copy -import time -import uuid - -from dogpile.cache import api -from dogpile.cache import proxy -import mock -from oslo_config import cfg - -from keystone.common import cache -from keystone import exception -from keystone.tests import unit - - -CONF = cfg.CONF -NO_VALUE = api.NO_VALUE - - -def _copy_value(value): - if value is not NO_VALUE: - value = copy.deepcopy(value) - return value - - -# NOTE(morganfainberg): WARNING - It is not recommended to use the Memory -# backend for dogpile.cache in a real deployment under any circumstances. The -# backend does no cleanup of expired values and therefore will leak memory. The -# backend is not implemented in a way to share data across processes (e.g. -# Keystone in HTTPD. This proxy is a hack to get around the lack of isolation -# of values in memory. Currently it blindly stores and retrieves the values -# from the cache, and modifications to dicts/lists/etc returned can result in -# changes to the cached values. In short, do not use the dogpile.cache.memory -# backend unless you are running tests or expecting odd/strange results. -class CacheIsolatingProxy(proxy.ProxyBackend): - """Proxy that forces a memory copy of stored values. - - The default in-memory cache-region does not perform a copy on values it is - meant to cache. Therefore if the value is modified after set or after get, - the cached value also is modified. This proxy does a copy as the last - thing before storing data. - - """ - def get(self, key): - return _copy_value(self.proxied.get(key)) - - def set(self, key, value): - self.proxied.set(key, _copy_value(value)) - - -class TestProxy(proxy.ProxyBackend): - def get(self, key): - value = _copy_value(self.proxied.get(key)) - if value is not NO_VALUE: - if isinstance(value[0], TestProxyValue): - value[0].cached = True - return value - - -class TestProxyValue(object): - def __init__(self, value): - self.value = value - self.cached = False - - -class CacheRegionTest(unit.TestCase): - - def setUp(self): - super(CacheRegionTest, self).setUp() - self.region = cache.make_region() - cache.configure_cache_region(self.region) - self.region.wrap(TestProxy) - self.test_value = TestProxyValue('Decorator Test') - - def _add_test_caching_option(self): - self.config_fixture.register_opt( - cfg.BoolOpt('caching', default=True), group='cache') - - def _get_cacheable_function(self): - with mock.patch.object(cache.REGION, 'cache_on_arguments', - self.region.cache_on_arguments): - memoize = cache.get_memoization_decorator(section='cache') - - @memoize - def cacheable_function(value): - return value - - return cacheable_function - - def test_region_built_with_proxy_direct_cache_test(self): - # Verify cache regions are properly built with proxies. - test_value = TestProxyValue('Direct Cache Test') - self.region.set('cache_test', test_value) - cached_value = self.region.get('cache_test') - self.assertTrue(cached_value.cached) - - def test_cache_region_no_error_multiple_config(self): - # Verify configuring the CacheRegion again doesn't error. - cache.configure_cache_region(self.region) - cache.configure_cache_region(self.region) - - def _get_cache_fallthrough_fn(self, cache_time): - with mock.patch.object(cache.REGION, 'cache_on_arguments', - self.region.cache_on_arguments): - memoize = cache.get_memoization_decorator( - section='cache', - expiration_section='assignment') - - class _test_obj(object): - def __init__(self, value): - self.test_value = value - - @memoize - def get_test_value(self): - return self.test_value - - def _do_test(value): - - test_obj = _test_obj(value) - - # Ensure the value has been cached - test_obj.get_test_value() - # Get the now cached value - cached_value = test_obj.get_test_value() - self.assertTrue(cached_value.cached) - self.assertEqual(value.value, cached_value.value) - self.assertEqual(cached_value.value, test_obj.test_value.value) - # Change the underlying value on the test object. - test_obj.test_value = TestProxyValue(uuid.uuid4().hex) - self.assertEqual(cached_value.value, - test_obj.get_test_value().value) - # override the system time to ensure the non-cached new value - # is returned - new_time = time.time() + (cache_time * 2) - with mock.patch.object(time, 'time', - return_value=new_time): - overriden_cache_value = test_obj.get_test_value() - self.assertNotEqual(cached_value.value, - overriden_cache_value.value) - self.assertEqual(test_obj.test_value.value, - overriden_cache_value.value) - - return _do_test - - def test_cache_no_fallthrough_expiration_time_fn(self): - # Since we do not re-configure the cache region, for ease of testing - # this value is set the same as the expiration_time default in the - # [cache] section - cache_time = 600 - expiration_time = cache.get_expiration_time_fn('role') - do_test = self._get_cache_fallthrough_fn(cache_time) - # Run the test with the assignment cache_time value - self.config_fixture.config(cache_time=cache_time, - group='role') - test_value = TestProxyValue(uuid.uuid4().hex) - self.assertEqual(cache_time, expiration_time()) - do_test(value=test_value) - - def test_cache_fallthrough_expiration_time_fn(self): - # Since we do not re-configure the cache region, for ease of testing - # this value is set the same as the expiration_time default in the - # [cache] section - cache_time = 599 - expiration_time = cache.get_expiration_time_fn('role') - do_test = self._get_cache_fallthrough_fn(cache_time) - # Run the test with the assignment cache_time value set to None and - # the global value set. - self.config_fixture.config(cache_time=None, group='role') - test_value = TestProxyValue(uuid.uuid4().hex) - self.assertIsNone(expiration_time()) - do_test(value=test_value) - - def test_should_cache_fn_global_cache_enabled(self): - # Verify should_cache_fn generates a sane function for subsystem and - # functions as expected with caching globally enabled. - cacheable_function = self._get_cacheable_function() - - self.config_fixture.config(group='cache', enabled=True) - cacheable_function(self.test_value) - cached_value = cacheable_function(self.test_value) - self.assertTrue(cached_value.cached) - - def test_should_cache_fn_global_cache_disabled(self): - # Verify should_cache_fn generates a sane function for subsystem and - # functions as expected with caching globally disabled. - cacheable_function = self._get_cacheable_function() - - self.config_fixture.config(group='cache', enabled=False) - cacheable_function(self.test_value) - cached_value = cacheable_function(self.test_value) - self.assertFalse(cached_value.cached) - - def test_should_cache_fn_global_cache_disabled_section_cache_enabled(self): - # Verify should_cache_fn generates a sane function for subsystem and - # functions as expected with caching globally disabled and the specific - # section caching enabled. - cacheable_function = self._get_cacheable_function() - - self._add_test_caching_option() - self.config_fixture.config(group='cache', enabled=False) - self.config_fixture.config(group='cache', caching=True) - - cacheable_function(self.test_value) - cached_value = cacheable_function(self.test_value) - self.assertFalse(cached_value.cached) - - def test_should_cache_fn_global_cache_enabled_section_cache_disabled(self): - # Verify should_cache_fn generates a sane function for subsystem and - # functions as expected with caching globally enabled and the specific - # section caching disabled. - cacheable_function = self._get_cacheable_function() - - self._add_test_caching_option() - self.config_fixture.config(group='cache', enabled=True) - self.config_fixture.config(group='cache', caching=False) - - cacheable_function(self.test_value) - cached_value = cacheable_function(self.test_value) - self.assertFalse(cached_value.cached) - - def test_should_cache_fn_global_cache_enabled_section_cache_enabled(self): - # Verify should_cache_fn generates a sane function for subsystem and - # functions as expected with caching globally enabled and the specific - # section caching enabled. - cacheable_function = self._get_cacheable_function() - - self._add_test_caching_option() - self.config_fixture.config(group='cache', enabled=True) - self.config_fixture.config(group='cache', caching=True) - - cacheable_function(self.test_value) - cached_value = cacheable_function(self.test_value) - self.assertTrue(cached_value.cached) - - def test_cache_dictionary_config_builder(self): - """Validate we build a sane dogpile.cache dictionary config.""" - self.config_fixture.config(group='cache', - config_prefix='test_prefix', - backend='some_test_backend', - expiration_time=86400, - backend_argument=['arg1:test', - 'arg2:test:test', - 'arg3.invalid']) - - config_dict = cache.build_cache_config() - self.assertEqual( - CONF.cache.backend, config_dict['test_prefix.backend']) - self.assertEqual( - CONF.cache.expiration_time, - config_dict['test_prefix.expiration_time']) - self.assertEqual('test', config_dict['test_prefix.arguments.arg1']) - self.assertEqual('test:test', - config_dict['test_prefix.arguments.arg2']) - self.assertNotIn('test_prefix.arguments.arg3', config_dict) - - def test_cache_debug_proxy(self): - single_value = 'Test Value' - single_key = 'testkey' - multi_values = {'key1': 1, 'key2': 2, 'key3': 3} - - self.region.set(single_key, single_value) - self.assertEqual(single_value, self.region.get(single_key)) - - self.region.delete(single_key) - self.assertEqual(NO_VALUE, self.region.get(single_key)) - - self.region.set_multi(multi_values) - cached_values = self.region.get_multi(multi_values.keys()) - for value in multi_values.values(): - self.assertIn(value, cached_values) - self.assertEqual(len(multi_values.values()), len(cached_values)) - - self.region.delete_multi(multi_values.keys()) - for value in self.region.get_multi(multi_values.keys()): - self.assertEqual(NO_VALUE, value) - - def test_configure_non_region_object_raises_error(self): - self.assertRaises(exception.ValidationError, - cache.configure_cache_region, - "bogus") - - -class CacheNoopBackendTest(unit.TestCase): - - def setUp(self): - super(CacheNoopBackendTest, self).setUp() - self.region = cache.make_region() - cache.configure_cache_region(self.region) - - def config_overrides(self): - super(CacheNoopBackendTest, self).config_overrides() - self.config_fixture.config(group='cache', - backend='keystone.common.cache.noop') - - def test_noop_backend(self): - single_value = 'Test Value' - single_key = 'testkey' - multi_values = {'key1': 1, 'key2': 2, 'key3': 3} - - self.region.set(single_key, single_value) - self.assertEqual(NO_VALUE, self.region.get(single_key)) - - self.region.set_multi(multi_values) - cached_values = self.region.get_multi(multi_values.keys()) - self.assertEqual(len(cached_values), len(multi_values.values())) - for value in cached_values: - self.assertEqual(NO_VALUE, value) - - # Delete should not raise exceptions - self.region.delete(single_key) - self.region.delete_multi(multi_values.keys()) diff --git a/keystone/tests/unit/test_cache_backend_mongo.py b/keystone/tests/unit/test_cache_backend_mongo.py deleted file mode 100644 index 66f80c21dc..0000000000 --- a/keystone/tests/unit/test_cache_backend_mongo.py +++ /dev/null @@ -1,728 +0,0 @@ -# Copyright 2014 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import collections -import copy -import functools -import uuid - -from dogpile.cache import api -from dogpile.cache import region as dp_region -import six -from six.moves import range - -from keystone.common.cache.backends import mongo -from keystone import exception -from keystone.tests import unit - - -# Mock database structure sample where 'ks_cache' is database and -# 'cache' is collection. Dogpile CachedValue data is divided in two -# fields `value` (CachedValue.payload) and `meta` (CachedValue.metadata) -ks_cache = { - "cache": [ - { - "value": { - "serviceType": "identity", - "allVersionsUrl": "https://dummyUrl", - "dateLastModified": "ISODDate(2014-02-08T18:39:13.237Z)", - "serviceName": "Identity", - "enabled": "True" - }, - "meta": { - "v": 1, - "ct": 1392371422.015121 - }, - "doc_date": "ISODate('2014-02-14T09:50:22.015Z')", - "_id": "8251dc95f63842719c077072f1047ddf" - }, - { - "value": "dummyValueX", - "meta": { - "v": 1, - "ct": 1392371422.014058 - }, - "doc_date": "ISODate('2014-02-14T09:50:22.014Z')", - "_id": "66730b9534d146f0804d23729ad35436" - } - ] -} - - -COLLECTIONS = {} -SON_MANIPULATOR = None - - -class MockCursor(object): - - def __init__(self, collection, dataset_factory): - super(MockCursor, self).__init__() - self.collection = collection - self._factory = dataset_factory - self._dataset = self._factory() - self._limit = None - self._skip = None - - def __iter__(self): - return self - - def __next__(self): - if self._skip: - for _ in range(self._skip): - next(self._dataset) - self._skip = None - if self._limit is not None and self._limit <= 0: - raise StopIteration() - if self._limit is not None: - self._limit -= 1 - return next(self._dataset) - - next = __next__ - - def __getitem__(self, index): - arr = [x for x in self._dataset] - self._dataset = iter(arr) - return arr[index] - - -class MockCollection(object): - - def __init__(self, db, name): - super(MockCollection, self).__init__() - self.name = name - self._collection_database = db - self._documents = {} - self.write_concern = {} - - def __getattr__(self, name): - if name == 'database': - return self._collection_database - - def ensure_index(self, key_or_list, *args, **kwargs): - pass - - def index_information(self): - return {} - - def find_one(self, spec_or_id=None, *args, **kwargs): - if spec_or_id is None: - spec_or_id = {} - if not isinstance(spec_or_id, collections.Mapping): - spec_or_id = {'_id': spec_or_id} - - try: - return next(self.find(spec_or_id, *args, **kwargs)) - except StopIteration: - return None - - def find(self, spec=None, *args, **kwargs): - return MockCursor(self, functools.partial(self._get_dataset, spec)) - - def _get_dataset(self, spec): - dataset = (self._copy_doc(document, dict) for document in - self._iter_documents(spec)) - return dataset - - def _iter_documents(self, spec=None): - return (SON_MANIPULATOR.transform_outgoing(document, self) for - document in six.itervalues(self._documents) - if self._apply_filter(document, spec)) - - def _apply_filter(self, document, query): - for key, search in query.items(): - doc_val = document.get(key) - if isinstance(search, dict): - op_dict = {'$in': lambda dv, sv: dv in sv} - is_match = all( - op_str in op_dict and op_dict[op_str](doc_val, search_val) - for op_str, search_val in search.items() - ) - else: - is_match = doc_val == search - - return is_match - - def _copy_doc(self, obj, container): - if isinstance(obj, list): - new = [] - for item in obj: - new.append(self._copy_doc(item, container)) - return new - if isinstance(obj, dict): - new = container() - for key, value in list(obj.items()): - new[key] = self._copy_doc(value, container) - return new - else: - return copy.copy(obj) - - def insert(self, data, manipulate=True, **kwargs): - if isinstance(data, list): - return [self._insert(element) for element in data] - return self._insert(data) - - def save(self, data, manipulate=True, **kwargs): - return self._insert(data) - - def _insert(self, data): - if '_id' not in data: - data['_id'] = uuid.uuid4().hex - object_id = data['_id'] - self._documents[object_id] = self._internalize_dict(data) - return object_id - - def find_and_modify(self, spec, document, upsert=False, **kwargs): - self.update(spec, document, upsert, **kwargs) - - def update(self, spec, document, upsert=False, **kwargs): - - existing_docs = [doc for doc in six.itervalues(self._documents) - if self._apply_filter(doc, spec)] - if existing_docs: - existing_doc = existing_docs[0] # should find only 1 match - _id = existing_doc['_id'] - existing_doc.clear() - existing_doc['_id'] = _id - existing_doc.update(self._internalize_dict(document)) - elif upsert: - existing_doc = self._documents[self._insert(document)] - - def _internalize_dict(self, d): - return {k: copy.deepcopy(v) for k, v in d.items()} - - def remove(self, spec_or_id=None, search_filter=None): - """Remove objects matching spec_or_id from the collection.""" - if spec_or_id is None: - spec_or_id = search_filter if search_filter else {} - if not isinstance(spec_or_id, dict): - spec_or_id = {'_id': spec_or_id} - to_delete = list(self.find(spec=spec_or_id)) - for doc in to_delete: - doc_id = doc['_id'] - del self._documents[doc_id] - - return { - "connectionId": uuid.uuid4().hex, - "n": len(to_delete), - "ok": 1.0, - "err": None, - } - - -class MockMongoDB(object): - def __init__(self, dbname): - self._dbname = dbname - self.mainpulator = None - - def authenticate(self, username, password): - pass - - def add_son_manipulator(self, manipulator): - global SON_MANIPULATOR - SON_MANIPULATOR = manipulator - - def __getattr__(self, name): - if name == 'authenticate': - return self.authenticate - elif name == 'name': - return self._dbname - elif name == 'add_son_manipulator': - return self.add_son_manipulator - else: - return get_collection(self._dbname, name) - - def __getitem__(self, name): - return get_collection(self._dbname, name) - - -class MockMongoClient(object): - def __init__(self, *args, **kwargs): - pass - - def __getattr__(self, dbname): - return MockMongoDB(dbname) - - -def get_collection(db_name, collection_name): - mongo_collection = MockCollection(MockMongoDB(db_name), collection_name) - return mongo_collection - - -def pymongo_override(): - global pymongo - import pymongo - if pymongo.MongoClient is not MockMongoClient: - pymongo.MongoClient = MockMongoClient - if pymongo.MongoReplicaSetClient is not MockMongoClient: - pymongo.MongoClient = MockMongoClient - - -class MyTransformer(mongo.BaseTransform): - """Added here just to check manipulator logic is used correctly.""" - - def transform_incoming(self, son, collection): - return super(MyTransformer, self).transform_incoming(son, collection) - - def transform_outgoing(self, son, collection): - return super(MyTransformer, self).transform_outgoing(son, collection) - - -class MongoCache(unit.BaseTestCase): - def setUp(self): - super(MongoCache, self).setUp() - global COLLECTIONS - COLLECTIONS = {} - mongo.MongoApi._DB = {} - mongo.MongoApi._MONGO_COLLS = {} - pymongo_override() - # using typical configuration - self.arguments = { - 'db_hosts': 'localhost:27017', - 'db_name': 'ks_cache', - 'cache_collection': 'cache', - 'username': 'test_user', - 'password': 'test_password' - } - - def test_missing_db_hosts(self): - self.arguments.pop('db_hosts') - region = dp_region.make_region() - self.assertRaises(exception.ValidationError, region.configure, - 'keystone.cache.mongo', - arguments=self.arguments) - - def test_missing_db_name(self): - self.arguments.pop('db_name') - region = dp_region.make_region() - self.assertRaises(exception.ValidationError, region.configure, - 'keystone.cache.mongo', - arguments=self.arguments) - - def test_missing_cache_collection_name(self): - self.arguments.pop('cache_collection') - region = dp_region.make_region() - self.assertRaises(exception.ValidationError, region.configure, - 'keystone.cache.mongo', - arguments=self.arguments) - - def test_incorrect_write_concern(self): - self.arguments['w'] = 'one value' - region = dp_region.make_region() - self.assertRaises(exception.ValidationError, region.configure, - 'keystone.cache.mongo', - arguments=self.arguments) - - def test_correct_write_concern(self): - self.arguments['w'] = 1 - region = dp_region.make_region().configure('keystone.cache.mongo', - arguments=self.arguments) - - random_key = uuid.uuid4().hex - region.set(random_key, "dummyValue10") - # There is no proxy so can access MongoCacheBackend directly - self.assertEqual(1, region.backend.api.w) - - def test_incorrect_read_preference(self): - self.arguments['read_preference'] = 'inValidValue' - region = dp_region.make_region().configure('keystone.cache.mongo', - arguments=self.arguments) - # As per delayed loading of pymongo, read_preference value should - # still be string and NOT enum - self.assertEqual('inValidValue', region.backend.api.read_preference) - - random_key = uuid.uuid4().hex - self.assertRaises(ValueError, region.set, - random_key, "dummyValue10") - - def test_correct_read_preference(self): - self.arguments['read_preference'] = 'secondaryPreferred' - region = dp_region.make_region().configure('keystone.cache.mongo', - arguments=self.arguments) - # As per delayed loading of pymongo, read_preference value should - # still be string and NOT enum - self.assertEqual('secondaryPreferred', - region.backend.api.read_preference) - - random_key = uuid.uuid4().hex - region.set(random_key, "dummyValue10") - - # Now as pymongo is loaded so expected read_preference value is enum. - # There is no proxy so can access MongoCacheBackend directly - self.assertEqual(3, region.backend.api.read_preference) - - def test_missing_replica_set_name(self): - self.arguments['use_replica'] = True - region = dp_region.make_region() - self.assertRaises(exception.ValidationError, region.configure, - 'keystone.cache.mongo', - arguments=self.arguments) - - def test_provided_replica_set_name(self): - self.arguments['use_replica'] = True - self.arguments['replicaset_name'] = 'my_replica' - dp_region.make_region().configure('keystone.cache.mongo', - arguments=self.arguments) - self.assertTrue(True) # reached here means no initialization error - - def test_incorrect_mongo_ttl_seconds(self): - self.arguments['mongo_ttl_seconds'] = 'sixty' - region = dp_region.make_region() - self.assertRaises(exception.ValidationError, region.configure, - 'keystone.cache.mongo', - arguments=self.arguments) - - def test_cache_configuration_values_assertion(self): - self.arguments['use_replica'] = True - self.arguments['replicaset_name'] = 'my_replica' - self.arguments['mongo_ttl_seconds'] = 60 - self.arguments['ssl'] = False - region = dp_region.make_region().configure('keystone.cache.mongo', - arguments=self.arguments) - # There is no proxy so can access MongoCacheBackend directly - self.assertEqual('localhost:27017', region.backend.api.hosts) - self.assertEqual('ks_cache', region.backend.api.db_name) - self.assertEqual('cache', region.backend.api.cache_collection) - self.assertEqual('test_user', region.backend.api.username) - self.assertEqual('test_password', region.backend.api.password) - self.assertEqual(True, region.backend.api.use_replica) - self.assertEqual('my_replica', region.backend.api.replicaset_name) - self.assertEqual(False, region.backend.api.conn_kwargs['ssl']) - self.assertEqual(60, region.backend.api.ttl_seconds) - - def test_multiple_region_cache_configuration(self): - arguments1 = copy.copy(self.arguments) - arguments1['cache_collection'] = 'cache_region1' - - region1 = dp_region.make_region().configure('keystone.cache.mongo', - arguments=arguments1) - # There is no proxy so can access MongoCacheBackend directly - self.assertEqual('localhost:27017', region1.backend.api.hosts) - self.assertEqual('ks_cache', region1.backend.api.db_name) - self.assertEqual('cache_region1', region1.backend.api.cache_collection) - self.assertEqual('test_user', region1.backend.api.username) - self.assertEqual('test_password', region1.backend.api.password) - # Should be None because of delayed initialization - self.assertIsNone(region1.backend.api._data_manipulator) - - random_key1 = uuid.uuid4().hex - region1.set(random_key1, "dummyValue10") - self.assertEqual("dummyValue10", region1.get(random_key1)) - # Now should have initialized - self.assertIsInstance(region1.backend.api._data_manipulator, - mongo.BaseTransform) - - class_name = '%s.%s' % (MyTransformer.__module__, "MyTransformer") - - arguments2 = copy.copy(self.arguments) - arguments2['cache_collection'] = 'cache_region2' - arguments2['son_manipulator'] = class_name - - region2 = dp_region.make_region().configure('keystone.cache.mongo', - arguments=arguments2) - # There is no proxy so can access MongoCacheBackend directly - self.assertEqual('localhost:27017', region2.backend.api.hosts) - self.assertEqual('ks_cache', region2.backend.api.db_name) - self.assertEqual('cache_region2', region2.backend.api.cache_collection) - - # Should be None because of delayed initialization - self.assertIsNone(region2.backend.api._data_manipulator) - - random_key = uuid.uuid4().hex - region2.set(random_key, "dummyValue20") - self.assertEqual("dummyValue20", region2.get(random_key)) - # Now should have initialized - self.assertIsInstance(region2.backend.api._data_manipulator, - MyTransformer) - - region1.set(random_key1, "dummyValue22") - self.assertEqual("dummyValue22", region1.get(random_key1)) - - def test_typical_configuration(self): - - dp_region.make_region().configure( - 'keystone.cache.mongo', - arguments=self.arguments - ) - self.assertTrue(True) # reached here means no initialization error - - def test_backend_get_missing_data(self): - - region = dp_region.make_region().configure( - 'keystone.cache.mongo', - arguments=self.arguments - ) - - random_key = uuid.uuid4().hex - # should return NO_VALUE as key does not exist in cache - self.assertEqual(api.NO_VALUE, region.get(random_key)) - - def test_backend_set_data(self): - - region = dp_region.make_region().configure( - 'keystone.cache.mongo', - arguments=self.arguments - ) - - random_key = uuid.uuid4().hex - region.set(random_key, "dummyValue") - self.assertEqual("dummyValue", region.get(random_key)) - - def test_backend_set_data_with_string_as_valid_ttl(self): - - self.arguments['mongo_ttl_seconds'] = '3600' - region = dp_region.make_region().configure('keystone.cache.mongo', - arguments=self.arguments) - self.assertEqual(3600, region.backend.api.ttl_seconds) - random_key = uuid.uuid4().hex - region.set(random_key, "dummyValue") - self.assertEqual("dummyValue", region.get(random_key)) - - def test_backend_set_data_with_int_as_valid_ttl(self): - - self.arguments['mongo_ttl_seconds'] = 1800 - region = dp_region.make_region().configure('keystone.cache.mongo', - arguments=self.arguments) - self.assertEqual(1800, region.backend.api.ttl_seconds) - random_key = uuid.uuid4().hex - region.set(random_key, "dummyValue") - self.assertEqual("dummyValue", region.get(random_key)) - - def test_backend_set_none_as_data(self): - - region = dp_region.make_region().configure( - 'keystone.cache.mongo', - arguments=self.arguments - ) - - random_key = uuid.uuid4().hex - region.set(random_key, None) - self.assertIsNone(region.get(random_key)) - - def test_backend_set_blank_as_data(self): - - region = dp_region.make_region().configure( - 'keystone.cache.mongo', - arguments=self.arguments - ) - - random_key = uuid.uuid4().hex - region.set(random_key, "") - self.assertEqual("", region.get(random_key)) - - def test_backend_set_same_key_multiple_times(self): - - region = dp_region.make_region().configure( - 'keystone.cache.mongo', - arguments=self.arguments - ) - - random_key = uuid.uuid4().hex - region.set(random_key, "dummyValue") - self.assertEqual("dummyValue", region.get(random_key)) - - dict_value = {'key1': 'value1'} - region.set(random_key, dict_value) - self.assertEqual(dict_value, region.get(random_key)) - - region.set(random_key, "dummyValue2") - self.assertEqual("dummyValue2", region.get(random_key)) - - def test_backend_multi_set_data(self): - - region = dp_region.make_region().configure( - 'keystone.cache.mongo', - arguments=self.arguments - ) - random_key = uuid.uuid4().hex - random_key1 = uuid.uuid4().hex - random_key2 = uuid.uuid4().hex - random_key3 = uuid.uuid4().hex - mapping = {random_key1: 'dummyValue1', - random_key2: 'dummyValue2', - random_key3: 'dummyValue3'} - region.set_multi(mapping) - # should return NO_VALUE as key does not exist in cache - self.assertEqual(api.NO_VALUE, region.get(random_key)) - self.assertFalse(region.get(random_key)) - self.assertEqual("dummyValue1", region.get(random_key1)) - self.assertEqual("dummyValue2", region.get(random_key2)) - self.assertEqual("dummyValue3", region.get(random_key3)) - - def test_backend_multi_get_data(self): - - region = dp_region.make_region().configure( - 'keystone.cache.mongo', - arguments=self.arguments - ) - random_key = uuid.uuid4().hex - random_key1 = uuid.uuid4().hex - random_key2 = uuid.uuid4().hex - random_key3 = uuid.uuid4().hex - mapping = {random_key1: 'dummyValue1', - random_key2: '', - random_key3: 'dummyValue3'} - region.set_multi(mapping) - - keys = [random_key, random_key1, random_key2, random_key3] - results = region.get_multi(keys) - # should return NO_VALUE as key does not exist in cache - self.assertEqual(api.NO_VALUE, results[0]) - self.assertEqual("dummyValue1", results[1]) - self.assertEqual("", results[2]) - self.assertEqual("dummyValue3", results[3]) - - def test_backend_multi_set_should_update_existing(self): - - region = dp_region.make_region().configure( - 'keystone.cache.mongo', - arguments=self.arguments - ) - random_key = uuid.uuid4().hex - random_key1 = uuid.uuid4().hex - random_key2 = uuid.uuid4().hex - random_key3 = uuid.uuid4().hex - mapping = {random_key1: 'dummyValue1', - random_key2: 'dummyValue2', - random_key3: 'dummyValue3'} - region.set_multi(mapping) - # should return NO_VALUE as key does not exist in cache - self.assertEqual(api.NO_VALUE, region.get(random_key)) - self.assertEqual("dummyValue1", region.get(random_key1)) - self.assertEqual("dummyValue2", region.get(random_key2)) - self.assertEqual("dummyValue3", region.get(random_key3)) - - mapping = {random_key1: 'dummyValue4', - random_key2: 'dummyValue5'} - region.set_multi(mapping) - self.assertEqual(api.NO_VALUE, region.get(random_key)) - self.assertEqual("dummyValue4", region.get(random_key1)) - self.assertEqual("dummyValue5", region.get(random_key2)) - self.assertEqual("dummyValue3", region.get(random_key3)) - - def test_backend_multi_set_get_with_blanks_none(self): - - region = dp_region.make_region().configure( - 'keystone.cache.mongo', - arguments=self.arguments - ) - random_key = uuid.uuid4().hex - random_key1 = uuid.uuid4().hex - random_key2 = uuid.uuid4().hex - random_key3 = uuid.uuid4().hex - random_key4 = uuid.uuid4().hex - mapping = {random_key1: 'dummyValue1', - random_key2: None, - random_key3: '', - random_key4: 'dummyValue4'} - region.set_multi(mapping) - # should return NO_VALUE as key does not exist in cache - self.assertEqual(api.NO_VALUE, region.get(random_key)) - self.assertEqual("dummyValue1", region.get(random_key1)) - self.assertIsNone(region.get(random_key2)) - self.assertEqual("", region.get(random_key3)) - self.assertEqual("dummyValue4", region.get(random_key4)) - - keys = [random_key, random_key1, random_key2, random_key3, random_key4] - results = region.get_multi(keys) - - # should return NO_VALUE as key does not exist in cache - self.assertEqual(api.NO_VALUE, results[0]) - self.assertEqual("dummyValue1", results[1]) - self.assertIsNone(results[2]) - self.assertEqual("", results[3]) - self.assertEqual("dummyValue4", results[4]) - - mapping = {random_key1: 'dummyValue5', - random_key2: 'dummyValue6'} - region.set_multi(mapping) - self.assertEqual(api.NO_VALUE, region.get(random_key)) - self.assertEqual("dummyValue5", region.get(random_key1)) - self.assertEqual("dummyValue6", region.get(random_key2)) - self.assertEqual("", region.get(random_key3)) - - def test_backend_delete_data(self): - - region = dp_region.make_region().configure( - 'keystone.cache.mongo', - arguments=self.arguments - ) - - random_key = uuid.uuid4().hex - region.set(random_key, "dummyValue") - self.assertEqual("dummyValue", region.get(random_key)) - - region.delete(random_key) - # should return NO_VALUE as key no longer exists in cache - self.assertEqual(api.NO_VALUE, region.get(random_key)) - - def test_backend_multi_delete_data(self): - - region = dp_region.make_region().configure( - 'keystone.cache.mongo', - arguments=self.arguments - ) - random_key = uuid.uuid4().hex - random_key1 = uuid.uuid4().hex - random_key2 = uuid.uuid4().hex - random_key3 = uuid.uuid4().hex - mapping = {random_key1: 'dummyValue1', - random_key2: 'dummyValue2', - random_key3: 'dummyValue3'} - region.set_multi(mapping) - # should return NO_VALUE as key does not exist in cache - self.assertEqual(api.NO_VALUE, region.get(random_key)) - self.assertEqual("dummyValue1", region.get(random_key1)) - self.assertEqual("dummyValue2", region.get(random_key2)) - self.assertEqual("dummyValue3", region.get(random_key3)) - self.assertEqual(api.NO_VALUE, region.get("InvalidKey")) - - keys = mapping.keys() - - region.delete_multi(keys) - - self.assertEqual(api.NO_VALUE, region.get("InvalidKey")) - # should return NO_VALUE as keys no longer exist in cache - self.assertEqual(api.NO_VALUE, region.get(random_key1)) - self.assertEqual(api.NO_VALUE, region.get(random_key2)) - self.assertEqual(api.NO_VALUE, region.get(random_key3)) - - def test_additional_crud_method_arguments_support(self): - """Additional arguments should works across find/insert/update.""" - - self.arguments['wtimeout'] = 30000 - self.arguments['j'] = True - self.arguments['continue_on_error'] = True - self.arguments['secondary_acceptable_latency_ms'] = 60 - region = dp_region.make_region().configure( - 'keystone.cache.mongo', - arguments=self.arguments - ) - - # There is no proxy so can access MongoCacheBackend directly - api_methargs = region.backend.api.meth_kwargs - self.assertEqual(30000, api_methargs['wtimeout']) - self.assertEqual(True, api_methargs['j']) - self.assertEqual(True, api_methargs['continue_on_error']) - self.assertEqual(60, api_methargs['secondary_acceptable_latency_ms']) - - random_key = uuid.uuid4().hex - region.set(random_key, "dummyValue1") - self.assertEqual("dummyValue1", region.get(random_key)) - - region.set(random_key, "dummyValue2") - self.assertEqual("dummyValue2", region.get(random_key)) - - random_key = uuid.uuid4().hex - region.set(random_key, "dummyValue3") - self.assertEqual("dummyValue3", region.get(random_key)) diff --git a/keystone/tests/unit/test_v3.py b/keystone/tests/unit/test_v3.py index 2665c5e564..38d3e5ac3b 100644 --- a/keystone/tests/unit/test_v3.py +++ b/keystone/tests/unit/test_v3.py @@ -168,7 +168,7 @@ class RestfulTestCase(unit.SQLDriverOverrides, rest.RestfulTestCase, def load_backends(self): # ensure the cache region instance is setup - cache.configure_cache_region(cache.REGION) + cache.configure_cache() super(RestfulTestCase, self).load_backends() diff --git a/keystone/token/persistence/core.py b/keystone/token/persistence/core.py index e68970ace5..434c17c979 100644 --- a/keystone/token/persistence/core.py +++ b/keystone/token/persistence/core.py @@ -32,9 +32,9 @@ from keystone.token import utils CONF = cfg.CONF LOG = log.getLogger(__name__) -MEMOIZE = cache.get_memoization_decorator(section='token') -REVOCATION_MEMOIZE = cache.get_memoization_decorator( - section='token', expiration_section='revoke') +MEMOIZE = cache.get_memoization_decorator(group='token') +REVOCATION_MEMOIZE = cache.get_memoization_decorator(group='token', + expiration_group='revoke') @dependency.requires('assignment_api', 'identity_api', 'resource_api', diff --git a/keystone/token/provider.py b/keystone/token/provider.py index 039a58e411..ffd691da85 100644 --- a/keystone/token/provider.py +++ b/keystone/token/provider.py @@ -38,7 +38,7 @@ from keystone.token import utils CONF = cfg.CONF LOG = log.getLogger(__name__) -MEMOIZE = cache.get_memoization_decorator(section='token') +MEMOIZE = cache.get_memoization_decorator(group='token') # NOTE(morganfainberg): This is for compatibility in case someone was relying # on the old location of the UnsupportedTokenVersionException for their code. diff --git a/requirements.txt b/requirements.txt index fa76a38b2d..dae149e9ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,6 +18,7 @@ stevedore>=1.5.0 # Apache-2.0 passlib>=1.6 python-keystoneclient>=1.6.0 keystonemiddleware>=2.0.0 +oslo.cache>=0.8.0 # Apache-2.0 oslo.concurrency>=2.3.0 # Apache-2.0 oslo.config>=2.3.0 # Apache-2.0 oslo.context>=0.2.0 # Apache-2.0 diff --git a/tox.ini b/tox.ini index 6579043118..9ad4fe371c 100644 --- a/tox.ini +++ b/tox.ini @@ -20,33 +20,43 @@ deps = -r{toxinidir}/test-requirements.txt commands = nosetests --with-coverage --cover-package=keystone \ keystone/tests/unit/auth/test_controllers.py \ + keystone/tests/unit/backend/domain_config/test_sql.py \ + keystone/tests/unit/backend/role/test_sql.py \ keystone/tests/unit/catalog/test_core.py \ keystone/tests/unit/common/test_injection.py \ keystone/tests/unit/common/test_json_home.py \ + keystone/tests/unit/common/test_manager.py \ keystone/tests/unit/common/test_sql_core.py \ keystone/tests/unit/common/test_utils.py \ + keystone/tests/unit/contrib/federation/test_utils.py \ + keystone/tests/unit/external/test_timeutils.py \ keystone/tests/unit/test_auth_plugin.py \ - keystone/tests/unit/test_backend.py \ keystone/tests/unit/test_backend_endpoint_policy.py \ + keystone/tests/unit/test_backend_endpoint_policy_sql.py \ + keystone/tests/unit/test_backend_federation_sql.py \ + keystone/tests/unit/test_backend_id_mapping_sql.py \ + keystone/tests/unit/test_backend.py \ keystone/tests/unit/test_backend_rules.py \ - keystone/tests/unit/test_cache_backend_mongo.py \ keystone/tests/unit/test_config.py \ + keystone/tests/unit/test_contrib_ec2.py \ keystone/tests/unit/test_contrib_s3_core.py \ keystone/tests/unit/test_driver_hints.py \ keystone/tests/unit/test_exception.py \ + keystone/tests/unit/test_ipv6.py \ keystone/tests/unit/test_policy.py \ keystone/tests/unit/test_singular_plural.py \ keystone/tests/unit/test_sql_livetest.py \ keystone/tests/unit/test_sql_migrate_extensions.py \ keystone/tests/unit/test_sql_upgrade.py \ keystone/tests/unit/test_ssl.py \ + keystone/tests/unit/tests/test_core.py \ + keystone/tests/unit/tests/test_utils.py \ keystone/tests/unit/test_token_bind.py \ keystone/tests/unit/test_url_middleware.py \ + keystone/tests/unit/test_v2_controller.py \ keystone/tests/unit/test_v3_controller.py \ keystone/tests/unit/test_validation.py \ keystone/tests/unit/test_wsgi.py \ - keystone/tests/unit/tests/test_core.py \ - keystone/tests/unit/tests/test_utils.py \ keystone/tests/unit/token/test_pki_provider.py \ keystone/tests/unit/token/test_pkiz_provider.py \ keystone/tests/unit/token/test_token_model.py \