Convert Token Memcache backend to new KeyValueStore Impl
This patchset converts the memcache backend for tokens to the new KeyValueStore implementation. This implementation provides basic configuration choices for utilizing memcache as a token backend. The test_backend_memcache test suite has been removed, as there is no extra functionality added on top of the KeyValueStore base besides the basic configuration choices. The test_backend_memcache test suite was only used for testing token memcache backend. The important unicode tests have been moved into the base token test suite and should now validate against all token backends. Closes-Bug: #1260080 bp: dogpile-kvs-backends Change-Id: I8f0bca409a6393176e77f7a27ba15959756da06f
This commit is contained in:
Morgan Fainberg
Gerrit Code Review
parent
57d02590f9
commit
813d1254eb
|
|
@ -1,257 +0,0 @@
|
|||
# Copyright 2012 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import copy
|
||||
import datetime
|
||||
import uuid
|
||||
|
||||
import memcache
|
||||
import six
|
||||
|
||||
from keystone.common import utils
|
||||
from keystone import config
|
||||
from keystone import exception
|
||||
from keystone.openstack.common import jsonutils
|
||||
from keystone.openstack.common import timeutils
|
||||
from keystone import tests
|
||||
from keystone.tests import test_backend
|
||||
from keystone.tests import test_utils
|
||||
from keystone import token
|
||||
from keystone.token.backends import memcache as token_memcache
|
||||
|
||||
CONF = config.CONF
|
||||
|
||||
|
||||
class MemcacheClient(object):
|
||||
"""Replicates a tiny subset of memcached client interface."""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
"""Ignores the passed in args."""
|
||||
self.cache = {}
|
||||
self.reject_cas = False
|
||||
|
||||
def add(self, key, value):
|
||||
if self.get(key):
|
||||
return False
|
||||
return self.set(key, value)
|
||||
|
||||
def append(self, key, value):
|
||||
existing_value = self.get(key)
|
||||
if existing_value:
|
||||
self.set(key, existing_value + value)
|
||||
return True
|
||||
return False
|
||||
|
||||
def check_key(self, key):
|
||||
if not isinstance(key, str):
|
||||
raise memcache.Client.MemcachedStringEncodingError()
|
||||
|
||||
def gets(self, key):
|
||||
#Call self.get() since we don't really do 'cas' here.
|
||||
return self.get(key)
|
||||
|
||||
def get(self, key):
|
||||
"""Retrieves the value for a key or None."""
|
||||
self.check_key(key)
|
||||
obj = self.cache.get(key)
|
||||
now = utils.unixtime(timeutils.utcnow())
|
||||
if obj and (obj[1] == 0 or obj[1] > now):
|
||||
# NOTE(morganfainberg): This behaves more like memcache
|
||||
# actually does and prevents modification of the passed in
|
||||
# reference from affecting the cached back-end data. This makes
|
||||
# tests a little easier to write.
|
||||
#
|
||||
# The back-end store should only change with an explicit
|
||||
# set/delete/append/etc
|
||||
data_copy = copy.deepcopy(obj[0])
|
||||
return data_copy
|
||||
|
||||
def set(self, key, value, time=0):
|
||||
"""Sets the value for a key."""
|
||||
self.check_key(key)
|
||||
# NOTE(morganfainberg): This behaves more like memcache
|
||||
# actually does and prevents modification of the passed in
|
||||
# reference from affecting the cached back-end data. This makes
|
||||
# tests a little easier to write.
|
||||
#
|
||||
# The back-end store should only change with an explicit
|
||||
# set/delete/append/etc
|
||||
data_copy = copy.deepcopy(value)
|
||||
self.cache[key] = (data_copy, time)
|
||||
return True
|
||||
|
||||
def cas(self, key, value, time=0, min_compress_len=0):
|
||||
# Call self.set() since we don't really do 'cas' here.
|
||||
if self.reject_cas:
|
||||
return False
|
||||
return self.set(key, value, time=time)
|
||||
|
||||
def reset_cas(self):
|
||||
#This is a stub for the memcache client reset_cas function.
|
||||
pass
|
||||
|
||||
def delete(self, key):
|
||||
self.check_key(key)
|
||||
try:
|
||||
del self.cache[key]
|
||||
except KeyError:
|
||||
#NOTE(bcwaldon): python-memcached always returns the same value
|
||||
pass
|
||||
|
||||
|
||||
class MemcacheToken(tests.TestCase, test_backend.TokenTests):
|
||||
def setUp(self):
|
||||
super(MemcacheToken, self).setUp()
|
||||
# Use the memcache backend for the token driver.
|
||||
self.opt_in_group('token',
|
||||
driver='keystone.token.backends.memcache.Token')
|
||||
self.load_backends()
|
||||
# Override the memcache client with the "dummy" client.
|
||||
fake_client = MemcacheClient()
|
||||
self.token_man = token.Manager()
|
||||
self.token_man.driver = token_memcache.Token(client=fake_client)
|
||||
self.token_api = self.token_man
|
||||
|
||||
def test_create_unicode_token_id(self):
|
||||
token_id = six.text_type(self._create_token_id())
|
||||
data = {'id': token_id, 'a': 'b',
|
||||
'user': {'id': 'testuserid'}}
|
||||
self.token_api.create_token(token_id, data)
|
||||
self.token_api.get_token(token_id)
|
||||
|
||||
def test_create_unicode_user_id(self):
|
||||
token_id = self._create_token_id()
|
||||
user_id = six.text_type(uuid.uuid4().hex)
|
||||
data = {'id': token_id, 'a': 'b',
|
||||
'user': {'id': user_id}}
|
||||
self.token_api.create_token(token_id, data)
|
||||
self.token_api.get_token(token_id)
|
||||
|
||||
def test_list_tokens_unicode_user_id(self):
|
||||
user_id = six.text_type(uuid.uuid4().hex)
|
||||
self.token_api.list_tokens(user_id)
|
||||
|
||||
def test_flush_expired_token(self):
|
||||
self.assertRaises(exception.NotImplemented,
|
||||
self.token_api.flush_expired_tokens)
|
||||
|
||||
def test_cleanup_user_index_on_create(self):
|
||||
valid_token_id = uuid.uuid4().hex
|
||||
second_valid_token_id = uuid.uuid4().hex
|
||||
expired_token_id = uuid.uuid4().hex
|
||||
user_id = six.text_type(uuid.uuid4().hex)
|
||||
|
||||
expire_delta = datetime.timedelta(seconds=CONF.token.expiration)
|
||||
|
||||
valid_data = {'id': valid_token_id, 'a': 'b',
|
||||
'user': {'id': user_id}}
|
||||
second_valid_data = {'id': second_valid_token_id, 'a': 'b',
|
||||
'user': {'id': user_id}}
|
||||
expired_data = {'id': expired_token_id, 'a': 'b',
|
||||
'user': {'id': user_id}}
|
||||
self.token_api.create_token(valid_token_id, valid_data)
|
||||
self.token_api.create_token(expired_token_id, expired_data)
|
||||
# NOTE(morganfainberg): Directly access the data cache since we need to
|
||||
# get expired tokens as well as valid tokens. token_api._list_tokens()
|
||||
# will not return any expired tokens in the list.
|
||||
user_key = self.token_api.driver._prefix_user_id(user_id)
|
||||
user_token_list = self.token_api.driver.client.get(user_key)
|
||||
self.assertEqual(len(user_token_list), 2)
|
||||
# user_token_list is a list of (token, expiry) tuples
|
||||
expired_idx = [i[0] for i in user_token_list].index(expired_token_id)
|
||||
# set the token as expired.
|
||||
user_token_list[expired_idx] = (user_token_list[expired_idx][0],
|
||||
timeutils.utcnow() - expire_delta)
|
||||
self.token_api.driver.client.set(user_key, user_token_list)
|
||||
|
||||
self.token_api.create_token(second_valid_token_id, second_valid_data)
|
||||
user_token_list = self.token_api.driver.client.get(user_key)
|
||||
self.assertEqual(len(user_token_list), 2)
|
||||
|
||||
def test_convert_token_list_from_json(self):
|
||||
token_list = ','.join(['"%s"' % uuid.uuid4().hex for x in xrange(5)])
|
||||
token_list_loaded = jsonutils.loads('[%s]' % token_list)
|
||||
converted_list = self.token_api.driver._convert_user_index_from_json(
|
||||
token_list, 'test-key')
|
||||
for idx, item in enumerate(converted_list):
|
||||
token_id, expiry = item
|
||||
self.assertEqual(token_id, token_list_loaded[idx])
|
||||
self.assertIsInstance(expiry, datetime.datetime)
|
||||
|
||||
def test_convert_token_list_from_json_non_string(self):
|
||||
token_list = self.token_api.driver._convert_user_index_from_json(
|
||||
None, 'test-key')
|
||||
self.assertEqual([], token_list)
|
||||
|
||||
def test_convert_token_list_from_json_invalid_json(self):
|
||||
token_list = self.token_api.driver._convert_user_index_from_json(
|
||||
'invalid_json_list', 'test-key')
|
||||
self.assertEqual([], token_list)
|
||||
|
||||
def test_cas_failure(self):
|
||||
expire_delta = datetime.timedelta(seconds=86400)
|
||||
self.token_api.driver.client.reject_cas = True
|
||||
token_id = uuid.uuid4().hex
|
||||
user_id = six.text_type(uuid.uuid4().hex)
|
||||
token_data = {'expires': timeutils.utcnow() + expire_delta,
|
||||
'id': token_id}
|
||||
user_key = self.token_api.driver._prefix_user_id(user_id)
|
||||
self.assertRaises(
|
||||
exception.UnexpectedError,
|
||||
self.token_api.driver._update_user_list_with_cas,
|
||||
user_key, token_id, token_data)
|
||||
|
||||
def test_token_expire_timezone(self):
|
||||
|
||||
@test_utils.timezone
|
||||
def _create_token(expire_time):
|
||||
token_id = uuid.uuid4().hex
|
||||
user_id = six.text_type(uuid.uuid4().hex)
|
||||
data = {'id': token_id, 'a': 'b', 'user': {'id': user_id},
|
||||
'expires': expire_time
|
||||
}
|
||||
self.token_api.create_token(token_id, data)
|
||||
return data
|
||||
|
||||
for d in ['+0', '-11', '-8', '-5', '+5', '+8', '+14']:
|
||||
test_utils.TZ = 'UTC' + d
|
||||
expire_time = timeutils.utcnow() + \
|
||||
datetime.timedelta(minutes=1)
|
||||
data_in = _create_token(expire_time)
|
||||
data_get = None
|
||||
data_get = self.token_api.get_token(data_in['id'])
|
||||
|
||||
self.assertIsNotNone(data_get, "TZ=%s" % test_utils.TZ)
|
||||
self.assertEqual(data_in['id'], data_get['id'],
|
||||
"TZ=%s" % test_utils.TZ)
|
||||
|
||||
expire_time_expired = timeutils.utcnow() + \
|
||||
datetime.timedelta(minutes=-1)
|
||||
data_in = _create_token(expire_time_expired)
|
||||
self.assertRaises(exception.TokenNotFound,
|
||||
self.token_api.get_token, data_in['id'])
|
||||
|
||||
|
||||
class MemcacheTokenCacheInvalidation(tests.TestCase,
|
||||
test_backend.TokenCacheInvalidation):
|
||||
def setUp(self):
|
||||
super(MemcacheTokenCacheInvalidation, self).setUp()
|
||||
CONF.token.driver = 'keystone.token.backends.memcache.Token'
|
||||
self.load_backends()
|
||||
fake_client = MemcacheClient()
|
||||
self.token_man = token.Manager()
|
||||
self.token_man.driver = token_memcache.Token(client=fake_client)
|
||||
self.token_api = self.token_man
|
||||
self.token_provider_api.driver.token_api = self.token_api
|
||||
self._create_test_data()
|
||||
|
|
@ -1,3 +1,6 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright 2013 Metacloud, Inc.
|
||||
# Copyright 2012 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
|
|
@ -12,260 +15,18 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from __future__ import absolute_import
|
||||
import copy
|
||||
import datetime
|
||||
|
||||
import memcache
|
||||
|
||||
from keystone.common import utils
|
||||
from keystone import config
|
||||
from keystone import exception
|
||||
from keystone.openstack.common import jsonutils
|
||||
from keystone.openstack.common import log
|
||||
from keystone.openstack.common import timeutils
|
||||
from keystone import token
|
||||
from keystone.common import config
|
||||
from keystone.token.backends import kvs
|
||||
|
||||
|
||||
CONF = config.CONF
|
||||
|
||||
LOG = log.getLogger(__name__)
|
||||
|
||||
class Token(kvs.Token):
|
||||
kvs_backend = 'openstack.kvs.Memcached'
|
||||
|
||||
class Token(token.Driver):
|
||||
revocation_key = 'revocation-list'
|
||||
|
||||
def __init__(self, client=None):
|
||||
self._memcache_client = client
|
||||
|
||||
@property
|
||||
def client(self):
|
||||
return self._memcache_client or self._get_memcache_client()
|
||||
|
||||
def _get_memcache_client(self):
|
||||
memcache_servers = CONF.memcache.servers
|
||||
# NOTE(morganfainberg): The memcache client library for python is NOT
|
||||
# thread safe and should not be passed between threads. This is highly
|
||||
# specific to the cas() (compare and set) methods and the caching of
|
||||
# the previous value(s). It appears greenthread should ensure there is
|
||||
# a single data structure per spawned greenthread.
|
||||
self._memcache_client = memcache.Client(memcache_servers, debug=0,
|
||||
cache_cas=True)
|
||||
return self._memcache_client
|
||||
|
||||
def _prefix_token_id(self, token_id):
|
||||
return 'token-%s' % token_id.encode('utf-8')
|
||||
|
||||
def _prefix_user_id(self, user_id):
|
||||
return 'usertokens-%s' % user_id.encode('utf-8')
|
||||
|
||||
def get_token(self, token_id):
|
||||
if token_id is None:
|
||||
raise exception.TokenNotFound(token_id='')
|
||||
ptk = self._prefix_token_id(token_id)
|
||||
token_ref = self.client.get(ptk)
|
||||
if token_ref is None:
|
||||
raise exception.TokenNotFound(token_id=token_id)
|
||||
|
||||
return token_ref
|
||||
|
||||
def create_token(self, token_id, data):
|
||||
data_copy = copy.deepcopy(data)
|
||||
ptk = self._prefix_token_id(token_id)
|
||||
if not data_copy.get('expires'):
|
||||
data_copy['expires'] = token.default_expire_time()
|
||||
if not data_copy.get('user_id'):
|
||||
data_copy['user_id'] = data_copy['user']['id']
|
||||
kwargs = {}
|
||||
if data_copy['expires'] is not None:
|
||||
expires_ts = utils.unixtime(data_copy['expires'])
|
||||
kwargs['time'] = expires_ts
|
||||
self.client.set(ptk, data_copy, **kwargs)
|
||||
if 'id' in data['user']:
|
||||
user_id = data['user']['id']
|
||||
user_key = self._prefix_user_id(user_id)
|
||||
# Append the new token_id to the token-index-list stored in the
|
||||
# user-key within memcache.
|
||||
self._update_user_list_with_cas(user_key, token_id, data_copy)
|
||||
return copy.deepcopy(data_copy)
|
||||
|
||||
def _convert_user_index_from_json(self, token_list, user_key):
|
||||
try:
|
||||
# NOTE(morganfainberg): Try loading in the old format
|
||||
# of the list.
|
||||
token_list = jsonutils.loads('[%s]' % token_list)
|
||||
|
||||
# NOTE(morganfainberg): Build a delta based upon the
|
||||
# token TTL configured. Since we are using the old
|
||||
# format index-list, we will create a "fake" expiration
|
||||
# that should be further in the future than the actual
|
||||
# expiry. To avoid locking up keystone trying to
|
||||
# communicate to memcached, it is better to use a fake
|
||||
# value. The logic that utilizes this list already
|
||||
# knows how to handle the case of tokens that are
|
||||
# no longer valid being included.
|
||||
delta = datetime.timedelta(
|
||||
seconds=CONF.token.expiration)
|
||||
new_expiry = timeutils.normalize_time(
|
||||
timeutils.utcnow()) + delta
|
||||
|
||||
for idx, token_id in enumerate(token_list):
|
||||
token_list[idx] = (token_id, new_expiry)
|
||||
|
||||
except Exception:
|
||||
# NOTE(morganfainberg): Catch any errors thrown here. There is
|
||||
# nothing the admin or operator needs to do in this case, but
|
||||
# it should be logged that there was an error and some action was
|
||||
# taken to correct it
|
||||
LOG.exception(_('Error converting user-token-index to new format; '
|
||||
'clearing user token index record "%s".'),
|
||||
user_key)
|
||||
token_list = []
|
||||
return token_list
|
||||
|
||||
def _update_user_list_with_cas(self, user_key, token_id, token_data):
|
||||
cas_retry = 0
|
||||
max_cas_retry = CONF.memcache.max_compare_and_set_retry
|
||||
current_time = timeutils.normalize_time(timeutils.utcnow())
|
||||
|
||||
self.client.reset_cas()
|
||||
|
||||
while cas_retry <= max_cas_retry:
|
||||
# NOTE(morganfainberg): cas or "compare and set" is a function of
|
||||
# memcache. It will return false if the value has changed since the
|
||||
# last call to client.gets(). This is the memcache supported method
|
||||
# of avoiding race conditions on set(). Memcache is already atomic
|
||||
# on the back-end and serializes operations.
|
||||
#
|
||||
# cas_retry is for tracking our iterations before we give up (in
|
||||
# case memcache is down or something horrible happens we don't
|
||||
# iterate forever trying to compare and set the new value.
|
||||
cas_retry += 1
|
||||
token_list = self.client.gets(user_key)
|
||||
filtered_list = []
|
||||
|
||||
if token_list is not None:
|
||||
if not isinstance(token_list, list):
|
||||
token_list = self._convert_user_index_from_json(token_list,
|
||||
user_key)
|
||||
for token_i, expiry in token_list:
|
||||
expires_at = timeutils.normalize_time(expiry)
|
||||
if expires_at < current_time:
|
||||
# skip tokens that are expired.
|
||||
continue
|
||||
|
||||
# Add the still valid token_id to the list.
|
||||
filtered_list.append((token_i, expiry))
|
||||
# Add the new token_id and expiry.
|
||||
filtered_list.append(
|
||||
(token_id, timeutils.normalize_time(token_data['expires'])))
|
||||
|
||||
# Use compare-and-set (cas) to set the new value for the
|
||||
# token-index-list for the user-key. Cas is used to prevent race
|
||||
# conditions from causing the loss of valid token ids from this
|
||||
# list.
|
||||
if self.client.cas(user_key, filtered_list):
|
||||
msg = _('Successful set of token-index-list for user-key '
|
||||
'"%(user_key)s", #%(count)d records')
|
||||
LOG.debug(msg, {'user_key': user_key,
|
||||
'count': len(filtered_list)})
|
||||
return filtered_list
|
||||
|
||||
# The cas function will return true if it succeeded or false if it
|
||||
# failed for any reason, including memcache server being down, cas
|
||||
# id changed since gets() called (the data changed between when
|
||||
# this loop started and this point, etc.
|
||||
error_msg = _('Failed to set token-index-list for user-key '
|
||||
'"%(user_key)s". Attempt %(cas_retry)d of '
|
||||
'%(cas_retry_max)d')
|
||||
LOG.debug(error_msg,
|
||||
{'user_key': user_key,
|
||||
'cas_retry': cas_retry,
|
||||
'cas_retry_max': max_cas_retry})
|
||||
|
||||
# Exceeded the maximum retry attempts.
|
||||
error_msg = _('Unable to add token user list')
|
||||
raise exception.UnexpectedError(error_msg)
|
||||
|
||||
def _add_to_revocation_list(self, data):
|
||||
data_json = jsonutils.dumps(data)
|
||||
if not self.client.append(self.revocation_key, ',%s' % data_json):
|
||||
if not self.client.add(self.revocation_key, data_json):
|
||||
if not self.client.append(self.revocation_key,
|
||||
',%s' % data_json):
|
||||
msg = _('Unable to add token to revocation list.')
|
||||
raise exception.UnexpectedError(msg)
|
||||
|
||||
def delete_token(self, token_id):
|
||||
# Test for existence
|
||||
data = self.get_token(token_id)
|
||||
ptk = self._prefix_token_id(token_id)
|
||||
result = self.client.delete(ptk)
|
||||
self._add_to_revocation_list(data)
|
||||
return result
|
||||
|
||||
def delete_tokens(self, user_id, tenant_id=None, trust_id=None,
|
||||
consumer_id=None):
|
||||
return super(Token, self).delete_tokens(
|
||||
user_id=user_id,
|
||||
tenant_id=tenant_id,
|
||||
trust_id=trust_id,
|
||||
consumer_id=consumer_id,
|
||||
)
|
||||
|
||||
def _list_tokens(self, user_id, tenant_id=None, trust_id=None,
|
||||
consumer_id=None):
|
||||
tokens = []
|
||||
user_key = self._prefix_user_id(user_id)
|
||||
current_time = timeutils.normalize_time(timeutils.utcnow())
|
||||
token_list = self.client.get(user_key) or []
|
||||
if not isinstance(token_list, list):
|
||||
# NOTE(morganfainberg): This is for compatibility for old-format
|
||||
# token-lists that were a JSON string of just token_ids. This code
|
||||
# will reference the underlying expires directly from the
|
||||
# token_ref vs in this list, so setting to none just ensures the
|
||||
# loop works as expected.
|
||||
token_list = [(i, None) for i in
|
||||
jsonutils.loads('[%s]' % token_list)]
|
||||
for token_id, expiry in token_list:
|
||||
ptk = self._prefix_token_id(token_id)
|
||||
token_ref = self.client.get(ptk)
|
||||
if token_ref:
|
||||
if tenant_id is not None:
|
||||
tenant = token_ref.get('tenant')
|
||||
if not tenant:
|
||||
continue
|
||||
if tenant.get('id') != tenant_id:
|
||||
continue
|
||||
if trust_id is not None:
|
||||
trust = token_ref.get('trust_id')
|
||||
if not trust:
|
||||
continue
|
||||
if trust != trust_id:
|
||||
continue
|
||||
if consumer_id is not None:
|
||||
try:
|
||||
oauth = token_ref['token_data']['token']['OS-OAUTH1']
|
||||
if oauth.get('consumer_id') != consumer_id:
|
||||
continue
|
||||
except KeyError:
|
||||
continue
|
||||
|
||||
if (timeutils.normalize_time(token_ref['expires']) <
|
||||
current_time):
|
||||
# Skip expired tokens.
|
||||
continue
|
||||
|
||||
tokens.append(token_id)
|
||||
return tokens
|
||||
|
||||
def list_revoked_tokens(self):
|
||||
list_json = self.client.get(self.revocation_key)
|
||||
if list_json:
|
||||
return jsonutils.loads('[%s]' % list_json)
|
||||
return []
|
||||
|
||||
def flush_expired_tokens(self):
|
||||
"""Archive or delete tokens that have expired.
|
||||
"""
|
||||
raise exception.NotImplemented()
|
||||
def __init__(self, *args, **kwargs):
|
||||
kwargs['no_expiry_keys'] = [self.revocation_key]
|
||||
kwargs['memcached_expire_time'] = CONF.token.expiration
|
||||
kwargs['url'] = CONF.memcache.servers
|
||||
super(Token, self).__init__(*args, **kwargs)
|
||||
|
|
|
|||
Loading…
Reference in New Issue