openstack/python-keystoneclient
Code Issues Proposed changes

Merge "Cache tokens using memorycache from oslo."

Browse Source
This commit is contained in:
Jenkins
2013-03-18 19:40:00 +00:00
committed by Gerrit Code Review
parent d62bbdfc39 6b9f55a483
commit b445c19dc6
3 changed files with 53 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
doc/source/middlewarearchitecture.rst
View File

@@ -196,15 +196,16 @@ Configuration Options
Caching for improved response Caching for improved response
----------------------------- -----------------------------
In order to prevent every service request, the middleware may be configured In order to prevent excessive requests and validations, the middleware uses an
to utilize a cache, and the keystone API returns the tokens with an in-memory cache for the tokens the keystone API returns. Keep in mind that
expiration (configurable in duration on the keystone service). The middleware invalidated tokens may continue to work if they are still in the token cache,
supports memcache based caching. so token_cache_time is configurable. For larger deployments, the middleware
also supports memcache based caching.
* ``memcache_servers``: (optonal) if defined, the memcache server(s) to use for * ``memcache_servers``: (optonal) if defined, the memcache server(s) to use for
cacheing. It will be ignored if Swift MemcacheRing is used instead. cacheing. It will be ignored if Swift MemcacheRing is used instead.
* ``token_cache_time``: (optional, default 300 seconds) Only valid if * ``token_cache_time``: (optional, default 300 seconds) Set to -1 to disable
memcache_servers is defined. caching completely.
When deploying auth_token middleware with Swift, user may elect When deploying auth_token middleware with Swift, user may elect
to use Swift MemcacheRing instead of the local Keystone memcache. to use Swift MemcacheRing instead of the local Keystone memcache.

15
keystoneclient/middleware/auth_token.py
View File

@@ -158,6 +158,7 @@ from keystoneclient.openstack.common import jsonutils
from keystoneclient.common import cms from keystoneclient.common import cms
from keystoneclient import utils from keystoneclient import utils
from keystoneclient.middleware import memcache_crypt from keystoneclient.middleware import memcache_crypt
from keystoneclient.openstack.common import memorycache
from keystoneclient.openstack.common import timeutils from keystoneclient.openstack.common import timeutils
CONF = None CONF = None
@@ -361,16 +362,8 @@ class AuthProtocol(object):
self._cache = env.get(cache) self._cache = env.get(cache)
else: else:
# use Keystone memcache # use Keystone memcache
memcache_servers = self._conf_get('memcache_servers') self._cache = memorycache.get_client(memcache_servers)
if memcache_servers:
try:
import memcache
self.LOG.info('Using Keystone memcache for caching token')
self._cache = memcache.Client(memcache_servers)
self._use_keystone_cache = True self._use_keystone_cache = True
except ImportError as e:
msg = 'disabled caching due to missing libraries %s' % (e)
self.LOG.warn(msg)
self._cache_initialized = True self._cache_initialized = True
def _conf_get(self, name): def _conf_get(self, name):
@@ -989,12 +982,8 @@ class AuthProtocol(object):
additional_headers=headers) additional_headers=headers)
if response.status == 200: if response.status == 200:
self._cache_put(user_token, data)
return data return data
if response.status == 404: if response.status == 404:
# FIXME(ja): I'm assuming the 404 status means that user_token is
# invalid - not that the admin_token is invalid
self._cache_store_invalid(user_token)
self.LOG.warn("Authorization failed for token %s", user_token) self.LOG.warn("Authorization failed for token %s", user_token)
raise InvalidUserToken('Token authorization failed') raise InvalidUserToken('Token authorization failed')
if response.status == 401: if response.status == 401:

128
tests/test_auth_token_middleware.py
View File

@@ -28,6 +28,7 @@ from keystoneclient.common import cms
from keystoneclient import utils from keystoneclient import utils
from keystoneclient.middleware import auth_token from keystoneclient.middleware import auth_token
from keystoneclient.middleware import memcache_crypt from keystoneclient.middleware import memcache_crypt
from keystoneclient.openstack.common import memorycache
from keystoneclient.openstack.common import jsonutils from keystoneclient.openstack.common import jsonutils
from keystoneclient.openstack.common import timeutils from keystoneclient.openstack.common import timeutils
from keystoneclient.middleware import test from keystoneclient.middleware import test
@@ -361,58 +362,11 @@ VERSION_LIST_v2 = {
} }
class FakeMemcache(object): class FakeSwiftMemcacheRing(memorycache.Client):
def __init__(self): # NOTE(vish): swift memcache uses param timeout instead of time
self.set_key = None def set(self, key, value, timeout=0, min_compress_len=0):
self.set_value = None sup = super(FakeSwiftMemcacheRing, self)
self.token_expiration = None sup.set(key, value, timeout, min_compress_len)
self.token_key = SIGNED_TOKEN_SCOPED_KEY
def get(self, key):
data = TOKEN_RESPONSES[self.token_key].copy()
if not data or key != "tokens/%s" % self.token_key:
return
if not self.token_expiration:
dt = datetime.datetime.now() + datetime.timedelta(minutes=5)
self.token_expiration = dt.strftime("%s")
return (data, str(self.token_expiration))
def set(self, key, value, time=None):
self.set_value = value
self.set_key = key
class v3FakeMemcache(FakeMemcache):
def __init__(self):
super(v3FakeMemcache, self).__init__()
self.token_key = SIGNED_v3_TOKEN_SCOPED_KEY
class FakeSwiftMemcacheRing(object):
def __init__(self):
self.set_key = None
self.set_value = None
self.token_expiration = None
self.token_key = SIGNED_TOKEN_SCOPED_KEY
def get(self, key):
data = TOKEN_RESPONSES[self.token_key].copy()
if not data or key != "tokens/%s" % self.token_key:
return
if not self.token_expiration:
dt = datetime.datetime.now() + datetime.timedelta(minutes=5)
self.token_expiration = dt.strftime("%s")
return (data, str(self.token_expiration))
def set(self, key, value, serialize=True, timeout=0):
self.set_value = value
self.set_key = key
class v3FakeSwiftMemcacheRing(FakeSwiftMemcacheRing):
def __init__(self):
super(v3FakeSwiftMemcacheRing, self).__init__()
self.token_key = SIGNED_v3_TOKEN_SCOPED_KEY
class FakeHTTPResponse(object): class FakeHTTPResponse(object):
@@ -606,8 +560,7 @@ class BaseAuthTokenMiddlewareTest(testtools.TestCase):
""" """
def setUp(self, expected_env=None, auth_version=None, def setUp(self, expected_env=None, auth_version=None,
fake_app=None, fake_http=None, token_dict=None, fake_app=None, fake_http=None, token_dict=None):
fake_memcache=None, fake_memcache_ring=None):
testtools.TestCase.setUp(self) testtools.TestCase.setUp(self)
expected_env = expected_env or {} expected_env = expected_env or {}
@@ -640,9 +593,6 @@ class BaseAuthTokenMiddlewareTest(testtools.TestCase):
self.response_status = None self.response_status = None
self.response_headers = None self.response_headers = None
self.fake_memcache = fake_memcache or FakeMemcache
self.fake_memcache_ring = fake_memcache_ring or FakeSwiftMemcacheRing
signed_list = 'SIGNED_REVOCATION_LIST' signed_list = 'SIGNED_REVOCATION_LIST'
valid_signed_list = 'VALID_SIGNED_REVOCATION_LIST' valid_signed_list = 'VALID_SIGNED_REVOCATION_LIST'
globals()[signed_list] = globals()[valid_signed_list] globals()[signed_list] = globals()[valid_signed_list]
@@ -978,44 +928,53 @@ class AuthTokenMiddlewareTest(test.NoModule, BaseAuthTokenMiddlewareTest):
self.assertEqual(self.response_headers['WWW-Authenticate'], self.assertEqual(self.response_headers['WWW-Authenticate'],
"Keystone uri='https://keystone.example.com:1234'") "Keystone uri='https://keystone.example.com:1234'")
def _get_cached_token(self, token):
token_id = cms.cms_hash_token(token)
# NOTE(vish): example tokens are expired so skip the expiration check.
key = self.middleware._get_cache_key(token_id)
cached = self.middleware._cache.get(key)
return self.middleware._unprotect_cache_value(token, cached)
def test_memcache(self): def test_memcache(self):
req = webob.Request.blank('/') req = webob.Request.blank('/')
req.headers['X-Auth-Token'] = ( token = self.token_dict['signed_token_scoped']
self.token_dict['signed_token_scoped']) req.headers['X-Auth-Token'] = token
self.middleware._cache = self.fake_memcache()
self.middleware._use_keystone_cache = True
self.middleware(req.environ, self.start_fake_response) self.middleware(req.environ, self.start_fake_response)
self.assertEqual(self.middleware._cache.set_value, None) self.assertNotEqual(self._get_cached_token(token), None)
def test_memcache_set_invalid(self): def test_memcache_set_invalid(self):
req = webob.Request.blank('/') req = webob.Request.blank('/')
req.headers['X-Auth-Token'] = 'invalid-token' token = 'invalid-token'
self.middleware._cache = self.fake_memcache() req.headers['X-Auth-Token'] = token
self.middleware._use_keystone_cache = True
self.middleware(req.environ, self.start_fake_response) self.middleware(req.environ, self.start_fake_response)
self.assertEqual(self.middleware._cache.set_value, "invalid") self.assertEqual(self._get_cached_token(token), "invalid")
def test_memcache_set_expired(self): def test_memcache_set_expired(self):
token_cache_time = 10
conf = {
'token_cache_time': token_cache_time,
'signing_dir': CERTDIR,
}
self.set_middleware(conf=conf)
req = webob.Request.blank('/') req = webob.Request.blank('/')
req.headers['X-Auth-Token'] = ( token = self.token_dict['signed_token_scoped']
self.token_dict['signed_token_scoped']) req.headers['X-Auth-Token'] = token
self.middleware._cache = self.fake_memcache() try:
self.middleware._use_keystone_cache = True now = datetime.datetime.utcnow()
expired = datetime.datetime.now() - datetime.timedelta(minutes=1) timeutils.set_time_override(now)
self.middleware._cache.token_expiration = float(expired.strftime("%s"))
self.middleware(req.environ, self.start_fake_response) self.middleware(req.environ, self.start_fake_response)
self.assertEqual(len(self.middleware._cache.set_value), 2) self.assertNotEqual(self._get_cached_token(token), None)
expired = now + datetime.timedelta(seconds=token_cache_time)
timeutils.set_time_override(expired)
self.assertEqual(self._get_cached_token(token), None)
finally:
timeutils.clear_time_override()
def test_swift_memcache_set_expired(self): def test_swift_memcache_set_expired(self):
req = webob.Request.blank('/') self.middleware._cache = FakeSwiftMemcacheRing()
req.headers['X-Auth-Token'] = (
self.token_dict['signed_token_scoped'])
self.middleware._cache = self.fake_memcache_ring()
self.middleware._use_keystone_cache = False self.middleware._use_keystone_cache = False
expired = datetime.datetime.now() - datetime.timedelta(minutes=1) self.middleware._cache_initialized = True
self.middleware._cache.token_expiration = float(expired.strftime("%s")) self.test_memcache_set_expired()
self.middleware(req.environ, self.start_fake_response)
self.assertEqual(len(self.middleware._cache.set_value), 2)
def test_nomemcache(self): def test_nomemcache(self):
self.disable_module('memcache') self.disable_module('memcache')
@@ -1042,7 +1001,6 @@ class AuthTokenMiddlewareTest(test.NoModule, BaseAuthTokenMiddlewareTest):
self.assertEqual(self.middleware._cache, 'CACHE_TEST') self.assertEqual(self.middleware._cache, 'CACHE_TEST')
def test_not_use_cache_from_env(self): def test_not_use_cache_from_env(self):
self.disable_module('memcache')
env = {'swift.cache': 'CACHE_TEST'} env = {'swift.cache': 'CACHE_TEST'}
conf = { conf = {
'auth_host': 'keystone.example.com', 'auth_host': 'keystone.example.com',
@@ -1052,7 +1010,7 @@ class AuthTokenMiddlewareTest(test.NoModule, BaseAuthTokenMiddlewareTest):
} }
self.set_middleware(conf=conf) self.set_middleware(conf=conf)
self.middleware._init_cache(env) self.middleware._init_cache(env)
self.assertEqual(self.middleware._cache, None) self.assertNotEqual(self.middleware._cache, 'CACHE_TEST')
def test_will_expire_soon(self): def test_will_expire_soon(self):
tenseconds = datetime.datetime.utcnow() + datetime.timedelta( tenseconds = datetime.datetime.utcnow() + datetime.timedelta(
@@ -1341,9 +1299,7 @@ class v3AuthTokenMiddlewareTest(AuthTokenMiddlewareTest):
auth_version='v3.0', auth_version='v3.0',
fake_app=v3FakeApp, fake_app=v3FakeApp,
fake_http=v3FakeHTTPConnection, fake_http=v3FakeHTTPConnection,
token_dict=token_dict, token_dict=token_dict)
fake_memcache=v3FakeMemcache,
fake_memcache_ring=v3FakeSwiftMemcacheRing)
def assert_valid_last_url(self, token_id): def assert_valid_last_url(self, token_id):
# Token ID is not part of the url in v3, so override # Token ID is not part of the url in v3, so override