Merge "Convert Token Memcache backend to new KeyValueStore Impl"
This commit is contained in:
commit
9fbb60dbe0
|
@ -1,257 +0,0 @@
|
|||
# Copyright 2012 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import copy
|
||||
import datetime
|
||||
import uuid
|
||||
|
||||
import memcache
|
||||
import six
|
||||
|
||||
from keystone.common import utils
|
||||
from keystone import config
|
||||
from keystone import exception
|
||||
from keystone.openstack.common import jsonutils
|
||||
from keystone.openstack.common import timeutils
|
||||
from keystone import tests
|
||||
from keystone.tests import test_backend
|
||||
from keystone.tests import test_utils
|
||||
from keystone import token
|
||||
from keystone.token.backends import memcache as token_memcache
|
||||
|
||||
CONF = config.CONF
|
||||
|
||||
|
||||
class MemcacheClient(object):
|
||||
"""Replicates a tiny subset of memcached client interface."""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
"""Ignores the passed in args."""
|
||||
self.cache = {}
|
||||
self.reject_cas = False
|
||||
|
||||
def add(self, key, value):
|
||||
if self.get(key):
|
||||
return False
|
||||
return self.set(key, value)
|
||||
|
||||
def append(self, key, value):
|
||||
existing_value = self.get(key)
|
||||
if existing_value:
|
||||
self.set(key, existing_value + value)
|
||||
return True
|
||||
return False
|
||||
|
||||
def check_key(self, key):
|
||||
if not isinstance(key, str):
|
||||
raise memcache.Client.MemcachedStringEncodingError()
|
||||
|
||||
def gets(self, key):
|
||||
#Call self.get() since we don't really do 'cas' here.
|
||||
return self.get(key)
|
||||
|
||||
def get(self, key):
|
||||
"""Retrieves the value for a key or None."""
|
||||
self.check_key(key)
|
||||
obj = self.cache.get(key)
|
||||
now = utils.unixtime(timeutils.utcnow())
|
||||
if obj and (obj[1] == 0 or obj[1] > now):
|
||||
# NOTE(morganfainberg): This behaves more like memcache
|
||||
# actually does and prevents modification of the passed in
|
||||
# reference from affecting the cached back-end data. This makes
|
||||
# tests a little easier to write.
|
||||
#
|
||||
# The back-end store should only change with an explicit
|
||||
# set/delete/append/etc
|
||||
data_copy = copy.deepcopy(obj[0])
|
||||
return data_copy
|
||||
|
||||
def set(self, key, value, time=0):
|
||||
"""Sets the value for a key."""
|
||||
self.check_key(key)
|
||||
# NOTE(morganfainberg): This behaves more like memcache
|
||||
# actually does and prevents modification of the passed in
|
||||
# reference from affecting the cached back-end data. This makes
|
||||
# tests a little easier to write.
|
||||
#
|
||||
# The back-end store should only change with an explicit
|
||||
# set/delete/append/etc
|
||||
data_copy = copy.deepcopy(value)
|
||||
self.cache[key] = (data_copy, time)
|
||||
return True
|
||||
|
||||
def cas(self, key, value, time=0, min_compress_len=0):
|
||||
# Call self.set() since we don't really do 'cas' here.
|
||||
if self.reject_cas:
|
||||
return False
|
||||
return self.set(key, value, time=time)
|
||||
|
||||
def reset_cas(self):
|
||||
#This is a stub for the memcache client reset_cas function.
|
||||
pass
|
||||
|
||||
def delete(self, key):
|
||||
self.check_key(key)
|
||||
try:
|
||||
del self.cache[key]
|
||||
except KeyError:
|
||||
#NOTE(bcwaldon): python-memcached always returns the same value
|
||||
pass
|
||||
|
||||
|
||||
class MemcacheToken(tests.TestCase, test_backend.TokenTests):
|
||||
def setUp(self):
|
||||
super(MemcacheToken, self).setUp()
|
||||
# Use the memcache backend for the token driver.
|
||||
self.opt_in_group('token',
|
||||
driver='keystone.token.backends.memcache.Token')
|
||||
self.load_backends()
|
||||
# Override the memcache client with the "dummy" client.
|
||||
fake_client = MemcacheClient()
|
||||
self.token_man = token.Manager()
|
||||
self.token_man.driver = token_memcache.Token(client=fake_client)
|
||||
self.token_api = self.token_man
|
||||
|
||||
def test_create_unicode_token_id(self):
|
||||
token_id = six.text_type(self._create_token_id())
|
||||
data = {'id': token_id, 'a': 'b',
|
||||
'user': {'id': 'testuserid'}}
|
||||
self.token_api.create_token(token_id, data)
|
||||
self.token_api.get_token(token_id)
|
||||
|
||||
def test_create_unicode_user_id(self):
|
||||
token_id = self._create_token_id()
|
||||
user_id = six.text_type(uuid.uuid4().hex)
|
||||
data = {'id': token_id, 'a': 'b',
|
||||
'user': {'id': user_id}}
|
||||
self.token_api.create_token(token_id, data)
|
||||
self.token_api.get_token(token_id)
|
||||
|
||||
def test_list_tokens_unicode_user_id(self):
|
||||
user_id = six.text_type(uuid.uuid4().hex)
|
||||
self.token_api.list_tokens(user_id)
|
||||
|
||||
def test_flush_expired_token(self):
|
||||
self.assertRaises(exception.NotImplemented,
|
||||
self.token_api.flush_expired_tokens)
|
||||
|
||||
def test_cleanup_user_index_on_create(self):
|
||||
valid_token_id = uuid.uuid4().hex
|
||||
second_valid_token_id = uuid.uuid4().hex
|
||||
expired_token_id = uuid.uuid4().hex
|
||||
user_id = six.text_type(uuid.uuid4().hex)
|
||||
|
||||
expire_delta = datetime.timedelta(seconds=CONF.token.expiration)
|
||||
|
||||
valid_data = {'id': valid_token_id, 'a': 'b',
|
||||
'user': {'id': user_id}}
|
||||
second_valid_data = {'id': second_valid_token_id, 'a': 'b',
|
||||
'user': {'id': user_id}}
|
||||
expired_data = {'id': expired_token_id, 'a': 'b',
|
||||
'user': {'id': user_id}}
|
||||
self.token_api.create_token(valid_token_id, valid_data)
|
||||
self.token_api.create_token(expired_token_id, expired_data)
|
||||
# NOTE(morganfainberg): Directly access the data cache since we need to
|
||||
# get expired tokens as well as valid tokens. token_api._list_tokens()
|
||||
# will not return any expired tokens in the list.
|
||||
user_key = self.token_api.driver._prefix_user_id(user_id)
|
||||
user_token_list = self.token_api.driver.client.get(user_key)
|
||||
self.assertEqual(len(user_token_list), 2)
|
||||
# user_token_list is a list of (token, expiry) tuples
|
||||
expired_idx = [i[0] for i in user_token_list].index(expired_token_id)
|
||||
# set the token as expired.
|
||||
user_token_list[expired_idx] = (user_token_list[expired_idx][0],
|
||||
timeutils.utcnow() - expire_delta)
|
||||
self.token_api.driver.client.set(user_key, user_token_list)
|
||||
|
||||
self.token_api.create_token(second_valid_token_id, second_valid_data)
|
||||
user_token_list = self.token_api.driver.client.get(user_key)
|
||||
self.assertEqual(len(user_token_list), 2)
|
||||
|
||||
def test_convert_token_list_from_json(self):
|
||||
token_list = ','.join(['"%s"' % uuid.uuid4().hex for x in xrange(5)])
|
||||
token_list_loaded = jsonutils.loads('[%s]' % token_list)
|
||||
converted_list = self.token_api.driver._convert_user_index_from_json(
|
||||
token_list, 'test-key')
|
||||
for idx, item in enumerate(converted_list):
|
||||
token_id, expiry = item
|
||||
self.assertEqual(token_id, token_list_loaded[idx])
|
||||
self.assertIsInstance(expiry, datetime.datetime)
|
||||
|
||||
def test_convert_token_list_from_json_non_string(self):
|
||||
token_list = self.token_api.driver._convert_user_index_from_json(
|
||||
None, 'test-key')
|
||||
self.assertEqual([], token_list)
|
||||
|
||||
def test_convert_token_list_from_json_invalid_json(self):
|
||||
token_list = self.token_api.driver._convert_user_index_from_json(
|
||||
'invalid_json_list', 'test-key')
|
||||
self.assertEqual([], token_list)
|
||||
|
||||
def test_cas_failure(self):
|
||||
expire_delta = datetime.timedelta(seconds=86400)
|
||||
self.token_api.driver.client.reject_cas = True
|
||||
token_id = uuid.uuid4().hex
|
||||
user_id = six.text_type(uuid.uuid4().hex)
|
||||
token_data = {'expires': timeutils.utcnow() + expire_delta,
|
||||
'id': token_id}
|
||||
user_key = self.token_api.driver._prefix_user_id(user_id)
|
||||
self.assertRaises(
|
||||
exception.UnexpectedError,
|
||||
self.token_api.driver._update_user_list_with_cas,
|
||||
user_key, token_id, token_data)
|
||||
|
||||
def test_token_expire_timezone(self):
|
||||
|
||||
@test_utils.timezone
|
||||
def _create_token(expire_time):
|
||||
token_id = uuid.uuid4().hex
|
||||
user_id = six.text_type(uuid.uuid4().hex)
|
||||
data = {'id': token_id, 'a': 'b', 'user': {'id': user_id},
|
||||
'expires': expire_time
|
||||
}
|
||||
self.token_api.create_token(token_id, data)
|
||||
return data
|
||||
|
||||
for d in ['+0', '-11', '-8', '-5', '+5', '+8', '+14']:
|
||||
test_utils.TZ = 'UTC' + d
|
||||
expire_time = timeutils.utcnow() + \
|
||||
datetime.timedelta(minutes=1)
|
||||
data_in = _create_token(expire_time)
|
||||
data_get = None
|
||||
data_get = self.token_api.get_token(data_in['id'])
|
||||
|
||||
self.assertIsNotNone(data_get, "TZ=%s" % test_utils.TZ)
|
||||
self.assertEqual(data_in['id'], data_get['id'],
|
||||
"TZ=%s" % test_utils.TZ)
|
||||
|
||||
expire_time_expired = timeutils.utcnow() + \
|
||||
datetime.timedelta(minutes=-1)
|
||||
data_in = _create_token(expire_time_expired)
|
||||
self.assertRaises(exception.TokenNotFound,
|
||||
self.token_api.get_token, data_in['id'])
|
||||
|
||||
|
||||
class MemcacheTokenCacheInvalidation(tests.TestCase,
|
||||
test_backend.TokenCacheInvalidation):
|
||||
def setUp(self):
|
||||
super(MemcacheTokenCacheInvalidation, self).setUp()
|
||||
CONF.token.driver = 'keystone.token.backends.memcache.Token'
|
||||
self.load_backends()
|
||||
fake_client = MemcacheClient()
|
||||
self.token_man = token.Manager()
|
||||
self.token_man.driver = token_memcache.Token(client=fake_client)
|
||||
self.token_api = self.token_man
|
||||
self.token_provider_api.driver.token_api = self.token_api
|
||||
self._create_test_data()
|
|
@ -1,3 +1,6 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2013 Metacloud, Inc.
|
||||
# Copyright 2012 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
|
@ -12,260 +15,18 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from __future__ import absolute_import
|
||||
import copy
|
||||
import datetime
|
||||
|
||||
import memcache
|
||||
|
||||
from keystone.common import utils
|
||||
from keystone import config
|
||||
from keystone import exception
|
||||
from keystone.openstack.common import jsonutils
|
||||
from keystone.openstack.common import log
|
||||
from keystone.openstack.common import timeutils
|
||||
from keystone import token
|
||||
from keystone.common import config
|
||||
from keystone.token.backends import kvs
|
||||
|
||||
|
||||
CONF = config.CONF
|
||||
|
||||
LOG = log.getLogger(__name__)
|
||||
|
||||
class Token(kvs.Token):
|
||||
kvs_backend = 'openstack.kvs.Memcached'
|
||||
|
||||
class Token(token.Driver):
|
||||
revocation_key = 'revocation-list'
|
||||
|
||||
def __init__(self, client=None):
|
||||
self._memcache_client = client
|
||||
|
||||
@property
|
||||
def client(self):
|
||||
return self._memcache_client or self._get_memcache_client()
|
||||
|
||||
def _get_memcache_client(self):
|
||||
memcache_servers = CONF.memcache.servers
|
||||
# NOTE(morganfainberg): The memcache client library for python is NOT
|
||||
# thread safe and should not be passed between threads. This is highly
|
||||
# specific to the cas() (compare and set) methods and the caching of
|
||||
# the previous value(s). It appears greenthread should ensure there is
|
||||
# a single data structure per spawned greenthread.
|
||||
self._memcache_client = memcache.Client(memcache_servers, debug=0,
|
||||
cache_cas=True)
|
||||
return self._memcache_client
|
||||
|
||||
def _prefix_token_id(self, token_id):
|
||||
return 'token-%s' % token_id.encode('utf-8')
|
||||
|
||||
def _prefix_user_id(self, user_id):
|
||||
return 'usertokens-%s' % user_id.encode('utf-8')
|
||||
|
||||
def get_token(self, token_id):
|
||||
if token_id is None:
|
||||
raise exception.TokenNotFound(token_id='')
|
||||
ptk = self._prefix_token_id(token_id)
|
||||
token_ref = self.client.get(ptk)
|
||||
if token_ref is None:
|
||||
raise exception.TokenNotFound(token_id=token_id)
|
||||
|
||||
return token_ref
|
||||
|
||||
def create_token(self, token_id, data):
|
||||
data_copy = copy.deepcopy(data)
|
||||
ptk = self._prefix_token_id(token_id)
|
||||
if not data_copy.get('expires'):
|
||||
data_copy['expires'] = token.default_expire_time()
|
||||
if not data_copy.get('user_id'):
|
||||
data_copy['user_id'] = data_copy['user']['id']
|
||||
kwargs = {}
|
||||
if data_copy['expires'] is not None:
|
||||
expires_ts = utils.unixtime(data_copy['expires'])
|
||||
kwargs['time'] = expires_ts
|
||||
self.client.set(ptk, data_copy, **kwargs)
|
||||
if 'id' in data['user']:
|
||||
user_id = data['user']['id']
|
||||
user_key = self._prefix_user_id(user_id)
|
||||
# Append the new token_id to the token-index-list stored in the
|
||||
# user-key within memcache.
|
||||
self._update_user_list_with_cas(user_key, token_id, data_copy)
|
||||
return copy.deepcopy(data_copy)
|
||||
|
||||
def _convert_user_index_from_json(self, token_list, user_key):
|
||||
try:
|
||||
# NOTE(morganfainberg): Try loading in the old format
|
||||
# of the list.
|
||||
token_list = jsonutils.loads('[%s]' % token_list)
|
||||
|
||||
# NOTE(morganfainberg): Build a delta based upon the
|
||||
# token TTL configured. Since we are using the old
|
||||
# format index-list, we will create a "fake" expiration
|
||||
# that should be further in the future than the actual
|
||||
# expiry. To avoid locking up keystone trying to
|
||||
# communicate to memcached, it is better to use a fake
|
||||
# value. The logic that utilizes this list already
|
||||
# knows how to handle the case of tokens that are
|
||||
# no longer valid being included.
|
||||
delta = datetime.timedelta(
|
||||
seconds=CONF.token.expiration)
|
||||
new_expiry = timeutils.normalize_time(
|
||||
timeutils.utcnow()) + delta
|
||||
|
||||
for idx, token_id in enumerate(token_list):
|
||||
token_list[idx] = (token_id, new_expiry)
|
||||
|
||||
except Exception:
|
||||
# NOTE(morganfainberg): Catch any errors thrown here. There is
|
||||
# nothing the admin or operator needs to do in this case, but
|
||||
# it should be logged that there was an error and some action was
|
||||
# taken to correct it
|
||||
LOG.exception(_('Error converting user-token-index to new format; '
|
||||
'clearing user token index record "%s".'),
|
||||
user_key)
|
||||
token_list = []
|
||||
return token_list
|
||||
|
||||
def _update_user_list_with_cas(self, user_key, token_id, token_data):
|
||||
cas_retry = 0
|
||||
max_cas_retry = CONF.memcache.max_compare_and_set_retry
|
||||
current_time = timeutils.normalize_time(timeutils.utcnow())
|
||||
|
||||
self.client.reset_cas()
|
||||
|
||||
while cas_retry <= max_cas_retry:
|
||||
# NOTE(morganfainberg): cas or "compare and set" is a function of
|
||||
# memcache. It will return false if the value has changed since the
|
||||
# last call to client.gets(). This is the memcache supported method
|
||||
# of avoiding race conditions on set(). Memcache is already atomic
|
||||
# on the back-end and serializes operations.
|
||||
#
|
||||
# cas_retry is for tracking our iterations before we give up (in
|
||||
# case memcache is down or something horrible happens we don't
|
||||
# iterate forever trying to compare and set the new value.
|
||||
cas_retry += 1
|
||||
token_list = self.client.gets(user_key)
|
||||
filtered_list = []
|
||||
|
||||
if token_list is not None:
|
||||
if not isinstance(token_list, list):
|
||||
token_list = self._convert_user_index_from_json(token_list,
|
||||
user_key)
|
||||
for token_i, expiry in token_list:
|
||||
expires_at = timeutils.normalize_time(expiry)
|
||||
if expires_at < current_time:
|
||||
# skip tokens that are expired.
|
||||
continue
|
||||
|
||||
# Add the still valid token_id to the list.
|
||||
filtered_list.append((token_i, expiry))
|
||||
# Add the new token_id and expiry.
|
||||
filtered_list.append(
|
||||
(token_id, timeutils.normalize_time(token_data['expires'])))
|
||||
|
||||
# Use compare-and-set (cas) to set the new value for the
|
||||
# token-index-list for the user-key. Cas is used to prevent race
|
||||
# conditions from causing the loss of valid token ids from this
|
||||
# list.
|
||||
if self.client.cas(user_key, filtered_list):
|
||||
msg = _('Successful set of token-index-list for user-key '
|
||||
'"%(user_key)s", #%(count)d records')
|
||||
LOG.debug(msg, {'user_key': user_key,
|
||||
'count': len(filtered_list)})
|
||||
return filtered_list
|
||||
|
||||
# The cas function will return true if it succeeded or false if it
|
||||
# failed for any reason, including memcache server being down, cas
|
||||
# id changed since gets() called (the data changed between when
|
||||
# this loop started and this point, etc.
|
||||
error_msg = _('Failed to set token-index-list for user-key '
|
||||
'"%(user_key)s". Attempt %(cas_retry)d of '
|
||||
'%(cas_retry_max)d')
|
||||
LOG.debug(error_msg,
|
||||
{'user_key': user_key,
|
||||
'cas_retry': cas_retry,
|
||||
'cas_retry_max': max_cas_retry})
|
||||
|
||||
# Exceeded the maximum retry attempts.
|
||||
error_msg = _('Unable to add token user list')
|
||||
raise exception.UnexpectedError(error_msg)
|
||||
|
||||
def _add_to_revocation_list(self, data):
|
||||
data_json = jsonutils.dumps(data)
|
||||
if not self.client.append(self.revocation_key, ',%s' % data_json):
|
||||
if not self.client.add(self.revocation_key, data_json):
|
||||
if not self.client.append(self.revocation_key,
|
||||
',%s' % data_json):
|
||||
msg = _('Unable to add token to revocation list.')
|
||||
raise exception.UnexpectedError(msg)
|
||||
|
||||
def delete_token(self, token_id):
|
||||
# Test for existence
|
||||
data = self.get_token(token_id)
|
||||
ptk = self._prefix_token_id(token_id)
|
||||
result = self.client.delete(ptk)
|
||||
self._add_to_revocation_list(data)
|
||||
return result
|
||||
|
||||
def delete_tokens(self, user_id, tenant_id=None, trust_id=None,
|
||||
consumer_id=None):
|
||||
return super(Token, self).delete_tokens(
|
||||
user_id=user_id,
|
||||
tenant_id=tenant_id,
|
||||
trust_id=trust_id,
|
||||
consumer_id=consumer_id,
|
||||
)
|
||||
|
||||
def _list_tokens(self, user_id, tenant_id=None, trust_id=None,
|
||||
consumer_id=None):
|
||||
tokens = []
|
||||
user_key = self._prefix_user_id(user_id)
|
||||
current_time = timeutils.normalize_time(timeutils.utcnow())
|
||||
token_list = self.client.get(user_key) or []
|
||||
if not isinstance(token_list, list):
|
||||
# NOTE(morganfainberg): This is for compatibility for old-format
|
||||
# token-lists that were a JSON string of just token_ids. This code
|
||||
# will reference the underlying expires directly from the
|
||||
# token_ref vs in this list, so setting to none just ensures the
|
||||
# loop works as expected.
|
||||
token_list = [(i, None) for i in
|
||||
jsonutils.loads('[%s]' % token_list)]
|
||||
for token_id, expiry in token_list:
|
||||
ptk = self._prefix_token_id(token_id)
|
||||
token_ref = self.client.get(ptk)
|
||||
if token_ref:
|
||||
if tenant_id is not None:
|
||||
tenant = token_ref.get('tenant')
|
||||
if not tenant:
|
||||
continue
|
||||
if tenant.get('id') != tenant_id:
|
||||
continue
|
||||
if trust_id is not None:
|
||||
trust = token_ref.get('trust_id')
|
||||
if not trust:
|
||||
continue
|
||||
if trust != trust_id:
|
||||
continue
|
||||
if consumer_id is not None:
|
||||
try:
|
||||
oauth = token_ref['token_data']['token']['OS-OAUTH1']
|
||||
if oauth.get('consumer_id') != consumer_id:
|
||||
continue
|
||||
except KeyError:
|
||||
continue
|
||||
|
||||
if (timeutils.normalize_time(token_ref['expires']) <
|
||||
current_time):
|
||||
# Skip expired tokens.
|
||||
continue
|
||||
|
||||
tokens.append(token_id)
|
||||
return tokens
|
||||
|
||||
def list_revoked_tokens(self):
|
||||
list_json = self.client.get(self.revocation_key)
|
||||
if list_json:
|
||||
return jsonutils.loads('[%s]' % list_json)
|
||||
return []
|
||||
|
||||
def flush_expired_tokens(self):
|
||||
"""Archive or delete tokens that have expired.
|
||||
"""
|
||||
raise exception.NotImplemented()
|
||||
def __init__(self, *args, **kwargs):
|
||||
kwargs['no_expiry_keys'] = [self.revocation_key]
|
||||
kwargs['memcached_expire_time'] = CONF.token.expiration
|
||||
kwargs['url'] = CONF.memcache.servers
|
||||
super(Token, self).__init__(*args, **kwargs)
|
||||
|
|
Loading…
Reference in New Issue