openstack/python-keystoneclient
Code Issues Proposed changes

Merge "Cache tokens using memorycache from oslo."

Browse Source
This commit is contained in:
Jenkins
2013-03-18 19:40:00 +00:00
committed by Gerrit Code Review
parent d62bbdfc39 6b9f55a483
commit b445c19dc6
3 changed files with 53 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
doc/source/middlewarearchitecture.rst
View File

@@ -196,15 +196,16 @@ Configuration Options
Caching for improved response
-----------------------------
In order to prevent every service request, the middleware may be configured
to utilize a cache, and the keystone API returns the tokens with an
expiration (configurable in duration on the keystone service). The middleware
supports memcache based caching.
In order to prevent excessive requests and validations, the middleware uses an
in-memory cache for the tokens the keystone API returns. Keep in mind that
invalidated tokens may continue to work if they are still in the token cache,
so token_cache_time is configurable. For larger deployments, the middleware
also supports memcache based caching.
* ``memcache_servers``: (optonal) if defined, the memcache server(s) to use for
cacheing. It will be ignored if Swift MemcacheRing is used instead.
* ``token_cache_time``: (optional, default 300 seconds) Only valid if
memcache_servers is defined.
* ``token_cache_time``: (optional, default 300 seconds) Set to -1 to disable
caching completely.
When deploying auth_token middleware with Swift, user may elect
to use Swift MemcacheRing instead of the local Keystone memcache.

15
keystoneclient/middleware/auth_token.py
View File

@@ -158,6 +158,7 @@ from keystoneclient.openstack.common import jsonutils
from keystoneclient.common import cms
from keystoneclient import utils
from keystoneclient.middleware import memcache_crypt
from keystoneclient.openstack.common import memorycache
from keystoneclient.openstack.common import timeutils
CONF = None
@@ -361,16 +362,8 @@ class AuthProtocol(object):
self._cache = env.get(cache)
else:
# use Keystone memcache
memcache_servers = self._conf_get('memcache_servers')
if memcache_servers:
try:
import memcache
self.LOG.info('Using Keystone memcache for caching token')
self._cache = memcache.Client(memcache_servers)
self._cache = memorycache.get_client(memcache_servers)
self._use_keystone_cache = True
except ImportError as e:
msg = 'disabled caching due to missing libraries %s' % (e)
self.LOG.warn(msg)
self._cache_initialized = True
def _conf_get(self, name):
@@ -989,12 +982,8 @@ class AuthProtocol(object):
additional_headers=headers)
if response.status == 200:
self._cache_put(user_token, data)
return data
if response.status == 404:
# FIXME(ja): I'm assuming the 404 status means that user_token is
# invalid - not that the admin_token is invalid
self._cache_store_invalid(user_token)
self.LOG.warn("Authorization failed for token %s", user_token)
raise InvalidUserToken('Token authorization failed')
if response.status == 401:

128
tests/test_auth_token_middleware.py
View File

@@ -28,6 +28,7 @@ from keystoneclient.common import cms
from keystoneclient import utils
from keystoneclient.middleware import auth_token
from keystoneclient.middleware import memcache_crypt
from keystoneclient.openstack.common import memorycache
from keystoneclient.openstack.common import jsonutils
from keystoneclient.openstack.common import timeutils
from keystoneclient.middleware import test
@@ -361,58 +362,11 @@ VERSION_LIST_v2 = {
}
class FakeMemcache(object):
def __init__(self):
self.set_key = None
self.set_value = None
self.token_expiration = None
self.token_key = SIGNED_TOKEN_SCOPED_KEY
def get(self, key):
data = TOKEN_RESPONSES[self.token_key].copy()
if not data or key != "tokens/%s" % self.token_key:
return
if not self.token_expiration:
dt = datetime.datetime.now() + datetime.timedelta(minutes=5)
self.token_expiration = dt.strftime("%s")
return (data, str(self.token_expiration))
def set(self, key, value, time=None):
self.set_value = value
self.set_key = key
class v3FakeMemcache(FakeMemcache):
def __init__(self):
super(v3FakeMemcache, self).__init__()
self.token_key = SIGNED_v3_TOKEN_SCOPED_KEY
class FakeSwiftMemcacheRing(object):
def __init__(self):
self.set_key = None
self.set_value = None
self.token_expiration = None
self.token_key = SIGNED_TOKEN_SCOPED_KEY
def get(self, key):
data = TOKEN_RESPONSES[self.token_key].copy()
if not data or key != "tokens/%s" % self.token_key:
return
if not self.token_expiration:
dt = datetime.datetime.now() + datetime.timedelta(minutes=5)
self.token_expiration = dt.strftime("%s")
return (data, str(self.token_expiration))
def set(self, key, value, serialize=True, timeout=0):
self.set_value = value
self.set_key = key
class v3FakeSwiftMemcacheRing(FakeSwiftMemcacheRing):
def __init__(self):
super(v3FakeSwiftMemcacheRing, self).__init__()
self.token_key = SIGNED_v3_TOKEN_SCOPED_KEY
class FakeSwiftMemcacheRing(memorycache.Client):
# NOTE(vish): swift memcache uses param timeout instead of time
def set(self, key, value, timeout=0, min_compress_len=0):
sup = super(FakeSwiftMemcacheRing, self)
sup.set(key, value, timeout, min_compress_len)
class FakeHTTPResponse(object):
@@ -606,8 +560,7 @@ class BaseAuthTokenMiddlewareTest(testtools.TestCase):
"""
def setUp(self, expected_env=None, auth_version=None,
fake_app=None, fake_http=None, token_dict=None,
fake_memcache=None, fake_memcache_ring=None):
fake_app=None, fake_http=None, token_dict=None):
testtools.TestCase.setUp(self)
expected_env = expected_env or {}
@@ -640,9 +593,6 @@ class BaseAuthTokenMiddlewareTest(testtools.TestCase):
self.response_status = None
self.response_headers = None
self.fake_memcache = fake_memcache or FakeMemcache
self.fake_memcache_ring = fake_memcache_ring or FakeSwiftMemcacheRing
signed_list = 'SIGNED_REVOCATION_LIST'
valid_signed_list = 'VALID_SIGNED_REVOCATION_LIST'
globals()[signed_list] = globals()[valid_signed_list]
@@ -978,44 +928,53 @@ class AuthTokenMiddlewareTest(test.NoModule, BaseAuthTokenMiddlewareTest):
self.assertEqual(self.response_headers['WWW-Authenticate'],
"Keystone uri='https://keystone.example.com:1234'")
def _get_cached_token(self, token):
token_id = cms.cms_hash_token(token)
# NOTE(vish): example tokens are expired so skip the expiration check.
key = self.middleware._get_cache_key(token_id)
cached = self.middleware._cache.get(key)
return self.middleware._unprotect_cache_value(token, cached)
def test_memcache(self):
req = webob.Request.blank('/')
req.headers['X-Auth-Token'] = (
self.token_dict['signed_token_scoped'])
self.middleware._cache = self.fake_memcache()
self.middleware._use_keystone_cache = True
token = self.token_dict['signed_token_scoped']
req.headers['X-Auth-Token'] = token
self.middleware(req.environ, self.start_fake_response)
self.assertEqual(self.middleware._cache.set_value, None)
self.assertNotEqual(self._get_cached_token(token), None)
def test_memcache_set_invalid(self):
req = webob.Request.blank('/')
req.headers['X-Auth-Token'] = 'invalid-token'
self.middleware._cache = self.fake_memcache()
self.middleware._use_keystone_cache = True
token = 'invalid-token'
req.headers['X-Auth-Token'] = token
self.middleware(req.environ, self.start_fake_response)
self.assertEqual(self.middleware._cache.set_value, "invalid")
self.assertEqual(self._get_cached_token(token), "invalid")
def test_memcache_set_expired(self):
token_cache_time = 10
conf = {
'token_cache_time': token_cache_time,
'signing_dir': CERTDIR,
}
self.set_middleware(conf=conf)
req = webob.Request.blank('/')
req.headers['X-Auth-Token'] = (
self.token_dict['signed_token_scoped'])
self.middleware._cache = self.fake_memcache()
self.middleware._use_keystone_cache = True
expired = datetime.datetime.now() - datetime.timedelta(minutes=1)
self.middleware._cache.token_expiration = float(expired.strftime("%s"))
token = self.token_dict['signed_token_scoped']
req.headers['X-Auth-Token'] = token
try:
now = datetime.datetime.utcnow()
timeutils.set_time_override(now)
self.middleware(req.environ, self.start_fake_response)
self.assertEqual(len(self.middleware._cache.set_value), 2)
self.assertNotEqual(self._get_cached_token(token), None)
expired = now + datetime.timedelta(seconds=token_cache_time)
timeutils.set_time_override(expired)
self.assertEqual(self._get_cached_token(token), None)
finally:
timeutils.clear_time_override()
def test_swift_memcache_set_expired(self):
req = webob.Request.blank('/')
req.headers['X-Auth-Token'] = (
self.token_dict['signed_token_scoped'])
self.middleware._cache = self.fake_memcache_ring()
self.middleware._cache = FakeSwiftMemcacheRing()
self.middleware._use_keystone_cache = False
expired = datetime.datetime.now() - datetime.timedelta(minutes=1)
self.middleware._cache.token_expiration = float(expired.strftime("%s"))
self.middleware(req.environ, self.start_fake_response)
self.assertEqual(len(self.middleware._cache.set_value), 2)
self.middleware._cache_initialized = True
self.test_memcache_set_expired()
def test_nomemcache(self):
self.disable_module('memcache')
@@ -1042,7 +1001,6 @@ class AuthTokenMiddlewareTest(test.NoModule, BaseAuthTokenMiddlewareTest):
self.assertEqual(self.middleware._cache, 'CACHE_TEST')
def test_not_use_cache_from_env(self):
self.disable_module('memcache')
env = {'swift.cache': 'CACHE_TEST'}
conf = {
'auth_host': 'keystone.example.com',
@@ -1052,7 +1010,7 @@ class AuthTokenMiddlewareTest(test.NoModule, BaseAuthTokenMiddlewareTest):
}
self.set_middleware(conf=conf)
self.middleware._init_cache(env)
self.assertEqual(self.middleware._cache, None)
self.assertNotEqual(self.middleware._cache, 'CACHE_TEST')
def test_will_expire_soon(self):
tenseconds = datetime.datetime.utcnow() + datetime.timedelta(
@@ -1341,9 +1299,7 @@ class v3AuthTokenMiddlewareTest(AuthTokenMiddlewareTest):
auth_version='v3.0',
fake_app=v3FakeApp,
fake_http=v3FakeHTTPConnection,
token_dict=token_dict,
fake_memcache=v3FakeMemcache,
fake_memcache_ring=v3FakeSwiftMemcacheRing)
token_dict=token_dict)
def assert_valid_last_url(self, token_id):
# Token ID is not part of the url in v3, so override