From 48764e6999c2b82998ff231584ae0c13c5017a3b Mon Sep 17 00:00:00 2001 From: Flavio Percoco Date: Tue, 21 Jan 2014 15:10:20 +0100 Subject: [PATCH] Sync oslo-cache from oslo-incubator This patch removes Marconi's local copy of the oslo-incubator cache work. The patch also removes the memcached backend, although the version coming from oslo doesn't have support for it yet. The backend will be pulled in again as soon as it is updated upstream. Change-Id: I487b2178d9657eb7abe020f06436b14531bdb993 --- marconi/common/cache/__init__.py | 13 - marconi/common/cache/_backends/__init__.py | 13 - marconi/common/cache/_backends/memcached.py | 105 ---- marconi/common/cache/_backends/memory.py | 87 ---- marconi/common/cache/backends.py | 188 -------- marconi/common/cache/cache.py | 61 --- marconi/openstack/common/cache/__init__.py | 0 .../common/cache/_backends/__init__.py | 0 .../common/cache/_backends/memory.py | 165 +++++++ marconi/openstack/common/cache/backends.py | 263 ++++++++++ marconi/openstack/common/cache/cache.py | 79 +++ marconi/openstack/common/excutils.py | 16 +- marconi/openstack/common/fileutils.py | 33 +- marconi/openstack/common/gettextutils.py | 448 ++++++++++-------- marconi/openstack/common/importutils.py | 2 - marconi/openstack/common/jsonutils.py | 17 +- marconi/openstack/common/local.py | 2 - marconi/openstack/common/lockutils.py | 126 +++-- marconi/openstack/common/log.py | 123 +++-- .../openstack/common/py3kcompat/__init__.py | 0 .../openstack/common/py3kcompat/urlutils.py | 67 +++ marconi/openstack/common/timeutils.py | 27 +- marconi/queues/bootstrap.py | 5 +- marconi/tests/queues/storage/base.py | 5 +- openstack-common.conf | 1 + setup.cfg | 5 +- .../unit/queues/storage/test_impl_mongodb.py | 2 +- .../unit/queues/storage/test_shard_catalog.py | 2 +- .../unit/queues/storage/test_shard_queues.py | 2 +- 29 files changed, 1064 insertions(+), 793 deletions(-) delete mode 100644 marconi/common/cache/__init__.py delete mode 100644 marconi/common/cache/_backends/__init__.py delete mode 100644 marconi/common/cache/_backends/memcached.py delete mode 100644 marconi/common/cache/_backends/memory.py delete mode 100644 marconi/common/cache/backends.py delete mode 100644 marconi/common/cache/cache.py create mode 100644 marconi/openstack/common/cache/__init__.py create mode 100644 marconi/openstack/common/cache/_backends/__init__.py create mode 100644 marconi/openstack/common/cache/_backends/memory.py create mode 100644 marconi/openstack/common/cache/backends.py create mode 100644 marconi/openstack/common/cache/cache.py create mode 100644 marconi/openstack/common/py3kcompat/__init__.py create mode 100644 marconi/openstack/common/py3kcompat/urlutils.py diff --git a/marconi/common/cache/__init__.py b/marconi/common/cache/__init__.py deleted file mode 100644 index e744db95d..000000000 --- a/marconi/common/cache/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2013 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. diff --git a/marconi/common/cache/_backends/__init__.py b/marconi/common/cache/_backends/__init__.py deleted file mode 100644 index e744db95d..000000000 --- a/marconi/common/cache/_backends/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2013 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. diff --git a/marconi/common/cache/_backends/memcached.py b/marconi/common/cache/_backends/memcached.py deleted file mode 100644 index bf0ce9a4b..000000000 --- a/marconi/common/cache/_backends/memcached.py +++ /dev/null @@ -1,105 +0,0 @@ -# Copyright 2013 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import time - -import memcache -from oslo.config import cfg - -from marconi.common.cache import backends - - -_memcache_opts = [ - cfg.ListOpt('memcached_servers', - default=['127.0.0.1:11211'], - help='Memcached servers or None for in process cache.'), -] - - -class MemcachedBackend(backends.BaseCache): - - def __init__(self, conf, group, cache_namespace): - conf.register_opts(_memcache_opts, group=group) - super(MemcachedBackend, self).__init__(conf, group, cache_namespace) - self._client = None - - @property - def _cache(self): - if not self._client: - self._client = memcache.Client(self.conf.memcached_servers) - return self._client - - def _get_ttl(self, ttl): - """Correct ttl for memcached.""" - - if ttl > 2592000: - # NOTE(flaper87): If ttl is bigger than 30 days, - # it needs to be translated to timestamp. - # - # See http://code.google.com/p/memcached/wiki/FAQ - # "You can set expire times up to 30 days in the - # future. After that memcached interprets it as a - # date, and will expire the item after said date. - # This is a simple (but obscure) mechanic." - return ttl + int(time.time()) - return ttl - - def set(self, key, value, ttl=0): - key = self._prepare_key(key) - return self._cache.set(key, value, self._get_ttl(ttl)) - - def unset(self, key): - self._cache.delete(self._prepare_key(key)) - - def get(self, key, default=None): - key = self._prepare_key(key) - value = self._cache.get(key) - return value is None and default or value - - def get_many(self, keys, default=None): - new_keys = map(self._prepare_key, keys) - ret = self._cache.get_multi(new_keys) - - m = dict(zip(new_keys, keys)) - for cache_key, key in m.items(): - yield (key, ret.get(cache_key, default)) - - def set_many(self, data, ttl=0): - safe_data = {} - for key, value in data.items(): - key = self._prepare_key(key) - safe_data[key] = value - self._cache.set_multi(safe_data, self._get_ttl(ttl)) - - def unset_many(self, keys, version=None): - self._cache.delete_multi(map(self._prepare_key, keys)) - - def incr(self, key, delta=1): - key = self._prepare_key(key) - - try: - if delta < 0: - #NOTE(flaper87): memcached doesn't support a negative delta - return self._cache.decr(key, -delta) - - return self._cache.incr(key, delta) - except ValueError: - return None - - def add(self, key, value, ttl=0): - key = self._prepare_key(key) - return self._cache.add(key, value, self._get_ttl(ttl)) - - def flush(self): - self._cache.flush_all() diff --git a/marconi/common/cache/_backends/memory.py b/marconi/common/cache/_backends/memory.py deleted file mode 100644 index 0d6c1c0ae..000000000 --- a/marconi/common/cache/_backends/memory.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2013 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -from marconi.common.cache import backends -from marconi.openstack.common import lockutils -from marconi.openstack.common import timeutils - - -class MemoryBackend(backends.BaseCache): - - def __init__(self, conf, group, cache_namespace): - super(MemoryBackend, self).__init__(conf, group, cache_namespace) - self._cache = {} - self._keys_expires = {} - - def set(self, key, value, ttl=0): - key = self._prepare_key(key) - with lockutils.lock(key): - expires_at = 0 - if ttl != 0: - expires_at = timeutils.utcnow_ts() + ttl - - self._cache[key] = (expires_at, value) - - if expires_at: - self._keys_expires.setdefault(expires_at, set()).add(key) - - return True - - def get(self, key, default=None): - key = self._prepare_key(key) - with lockutils.lock(key): - now = timeutils.utcnow_ts() - - try: - timeout, value = self._cache[key] - - if timeout and now >= timeout: - del self._cache[key] - return default - - return value - except KeyError: - return default - - def _purge_expired(self): - """Removes expired keys from the cache.""" - - now = timeutils.utcnow_ts() - for timeout in sorted(self._keys_expires.keys()): - - # NOTE(flaper87): If timeout is greater - # than `now`, stop the iteration, remaining - # keys have not expired. - if now < timeout: - break - - # NOTE(flaper87): Unset every key in - # this set from the cache if its timeout - # is equal to `timeout`. (They key might - # have been updated) - for subkey in self._keys_expires.pop(timeout): - if self._cache[subkey][0] == timeout: - del self._cache[subkey] - - def unset(self, key): - self._purge_expired() - - # NOTE(flaper87): Delete the key. Using pop - # since it could have been deleted already - self._cache.pop(self._prepare_key(key), None) - - def flush(self): - self._cache = {} - self._keys_expires = {} diff --git a/marconi/common/cache/backends.py b/marconi/common/cache/backends.py deleted file mode 100644 index 0f1b9f0bf..000000000 --- a/marconi/common/cache/backends.py +++ /dev/null @@ -1,188 +0,0 @@ -# Copyright 2013 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import abc -import six - - -@six.add_metaclass(abc.ABCMeta) -class BaseCache(object): - - def __init__(self, conf, group, cache_namespace): - self.conf = conf[group] - self._cache_namespace = cache_namespace - - @abc.abstractmethod - def set(self, key, value, ttl=0): - """Sets or updates a cache entry - - :params key: Item key as string. - :params value: Value to assign to the key. This - can be anything that is handled - by the current backend. - :params ttl: Key's timeout in seconds. - - :returns: True if the operation succeeds. - """ - - @abc.abstractmethod - def get(self, key, default=None): - """Gets one item from the cache - - :params key: Key for the item to retrieve - from the cache. - :params default: The default value to return. - - :returns: `key`'s value in the cache if it exists, - otherwise `default` should be returned. - """ - - @abc.abstractmethod - def unset(self, key): - """Removes an item from cache. - - :params key: The key to remove. - - :returns: The key value if there's one - """ - - def _prepare_key(self, key): - """Prepares the key - - This method concatenates the cache_namespace - and the key so it can be used in the cache. - - NOTE: All cache backends have to call it - explicitly where needed. - - :param key: The key to be prefixed - """ - prepared = key - if self._cache_namespace: - prepared = ("%(prefix)s-%(key)s" % - {'prefix': self._cache_namespace, 'key': key}) - - # NOTE(cpp-cabrera): some caching backends (memcache) enforce - # that the key type must be bytes. This is here to ensure that - # this precondition is respected and that users can continue - # to use the caching layer transparently. - if isinstance(prepared, six.text_type): - prepared = prepared.encode('utf8') - - return prepared - - def add(self, key, value, ttl=0): - """Sets the value for a key if it doesn't exist - - :params key: Key to create as string. - :params value: Value to assign to the key. This - can be anything that is handled - by current backend. - :params ttl: Key's timeout in seconds. - - :returns: False if the key exists, otherwise, - `set`'s result will be returned. - """ - - if self.get(key) is not None: - return False - return self.set(key, value, ttl) - - def get_many(self, keys, default=None): - """Gets key's value from cache - - :params keys: List of keys to retrieve. - :params default: The default value to return - for each key that is not in - the cache. - - :returns: A generator of (key, value) - """ - for k in keys: - val = self.get(k, default=default) - if val is not None: - yield (k, val) - - def has_key(self, key): - """Verifies that a key exists. - - :params key: The key to verify. - - :returns: True if the key exists, otherwise - False. - """ - return self.get(key) is not None - - def set_many(self, data, ttl=0): - """Puts several items into the cache at once - - Depending on the backend, this operation may or may - not be efficient. The default implementation calls - set for each (key, value) pair passed, other backends - support set_many operations as part of their protocols. - - :params data: A dictionary like {key: val} to store - in the cache. - :params ttl: Key's timeout in seconds. - """ - for key, value in data.items(): - self.set(key, value, ttl=ttl) - - def unset_many(self, keys): - """Removes several keys from the cache at once - - :params keys: List of keys to retrieve. - """ - for key in keys: - self.unset(key) - - def incr(self, key, delta=1): - """Increments the value for a key - - NOTE: This method is not synchronized because - get and set are. - - :params key: The key for the value to be incremented - :params delta: Number of units by which to increment - the value. Pass a negative number to - decrement the value. - - :returns: The new value - """ - value = self.get(key) - if value is None: - return None - new_value = value + delta - self.set(key, new_value) - return new_value - - def append(self, key, tail): - """Appends `value` to `key`'s value. - - :params key: The key of the value to which - `tail` should be appended. - :params tail: The value to append to the - original. - - :returns: The new value - """ - value = self.get(key) - if value is None: - return None - new_value = value + tail - self.set(key, new_value) - return new_value - - def flush(self): - """Flushes all items from the cache.""" diff --git a/marconi/common/cache/cache.py b/marconi/common/cache/cache.py deleted file mode 100644 index bc503dadc..000000000 --- a/marconi/common/cache/cache.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2013 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Cache library. - -Supported configuration options: - -`cache_backend`: Name of the cache backend to use. -""" - -from oslo.config import cfg -from stevedore import driver - - -_cache_options = [ - cfg.StrOpt('cache_backend', - default='memory', - help='The cache driver to use, default value is `memory`.'), - cfg.StrOpt('cache_prefix', - default=None, - help='Prefix to use in every cache key'), -] - - -def get_cache(conf): - """Loads the cache backend - - This function loads the cache backend - specified in the given configuration. - - :param conf: Configuration instance to use - """ - - # NOTE(flaper87): oslo.config checks if options - # exist before registering them. The code bellow - # should be safe. - cache_group = cfg.OptGroup(name='oslo_cache', - title='Cache options') - - conf.register_group(cache_group) - conf.register_opts(_cache_options, group=cache_group) - - kwargs = dict(cache_namespace=conf.oslo_cache.cache_prefix) - - backend = conf.oslo_cache.cache_backend - mgr = driver.DriverManager('marconi.common.cache.backends', backend, - invoke_on_load=True, - invoke_args=[conf, cache_group.name], - invoke_kwds=kwargs) - return mgr.driver diff --git a/marconi/openstack/common/cache/__init__.py b/marconi/openstack/common/cache/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/marconi/openstack/common/cache/_backends/__init__.py b/marconi/openstack/common/cache/_backends/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/marconi/openstack/common/cache/_backends/memory.py b/marconi/openstack/common/cache/_backends/memory.py new file mode 100644 index 000000000..1c3b0b21c --- /dev/null +++ b/marconi/openstack/common/cache/_backends/memory.py @@ -0,0 +1,165 @@ +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import collections + +from marconi.openstack.common.cache import backends +from marconi.openstack.common import lockutils +from marconi.openstack.common import timeutils + + +class MemoryBackend(backends.BaseCache): + + def __init__(self, parsed_url, options=None): + super(MemoryBackend, self).__init__(parsed_url, options) + self._clear() + + def _set_unlocked(self, key, value, ttl=0): + expires_at = 0 + if ttl != 0: + expires_at = timeutils.utcnow_ts() + ttl + + self._cache[key] = (expires_at, value) + + if expires_at: + self._keys_expires[expires_at].add(key) + + def _set(self, key, value, ttl=0, not_exists=False): + with lockutils.lock(key): + + # NOTE(flaper87): This is needed just in `set` + # calls, hence it's not in `_set_unlocked` + if not_exists and self._exists_unlocked(key): + return False + + self._set_unlocked(key, value, ttl) + return True + + def _get_unlocked(self, key, default=None): + now = timeutils.utcnow_ts() + + try: + timeout, value = self._cache[key] + except KeyError: + return (0, default) + + if timeout and now >= timeout: + + # NOTE(flaper87): Record expired, + # remove it from the cache but catch + # KeyError and ValueError in case + # _purge_expired removed this key already. + try: + del self._cache[key] + except KeyError: + pass + + try: + # NOTE(flaper87): Keys with ttl == 0 + # don't exist in the _keys_expires dict + self._keys_expires[timeout].remove(key) + except (KeyError, ValueError): + pass + + return (0, default) + + return (timeout, value) + + def _get(self, key, default=None): + with lockutils.lock(key): + return self._get_unlocked(key, default)[1] + + def _exists_unlocked(self, key): + now = timeutils.utcnow_ts() + try: + timeout = self._cache[key][0] + return not timeout or now <= timeout + except KeyError: + return False + + def __contains__(self, key): + with lockutils.lock(key): + return self._exists_unlocked(key) + + def _incr_append(self, key, other): + with lockutils.lock(key): + timeout, value = self._get_unlocked(key) + + if value is None: + return None + + ttl = timeutils.utcnow_ts() - timeout + new_value = value + other + self._set_unlocked(key, new_value, ttl) + return new_value + + def _incr(self, key, delta): + if not isinstance(delta, int): + raise TypeError('delta must be an int instance') + + return self._incr_append(key, delta) + + def _append_tail(self, key, tail): + return self._incr_append(key, tail) + + def _purge_expired(self): + """Removes expired keys from the cache.""" + + now = timeutils.utcnow_ts() + for timeout in sorted(self._keys_expires.keys()): + + # NOTE(flaper87): If timeout is greater + # than `now`, stop the iteration, remaining + # keys have not expired. + if now < timeout: + break + + # NOTE(flaper87): Unset every key in + # this set from the cache if its timeout + # is equal to `timeout`. (The key might + # have been updated) + for subkey in self._keys_expires.pop(timeout): + try: + if self._cache[subkey][0] == timeout: + del self._cache[subkey] + except KeyError: + continue + + def __delitem__(self, key): + self._purge_expired() + + # NOTE(flaper87): Delete the key. Using pop + # since it could have been deleted already + value = self._cache.pop(key, None) + + if value: + try: + # NOTE(flaper87): Keys with ttl == 0 + # don't exist in the _keys_expires dict + self._keys_expires[value[0]].remove(value[1]) + except (KeyError, ValueError): + pass + + def _clear(self): + self._cache = {} + self._keys_expires = collections.defaultdict(set) + + def _get_many(self, keys, default): + return super(MemoryBackend, self)._get_many(keys, default) + + def _set_many(self, data, ttl=0): + return super(MemoryBackend, self)._set_many(data, ttl) + + def _unset_many(self, keys): + return super(MemoryBackend, self)._unset_many(keys) diff --git a/marconi/openstack/common/cache/backends.py b/marconi/openstack/common/cache/backends.py new file mode 100644 index 000000000..2fa4aaeb2 --- /dev/null +++ b/marconi/openstack/common/cache/backends.py @@ -0,0 +1,263 @@ +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc + +import six + + +NOTSET = object() + + +@six.add_metaclass(abc.ABCMeta) +class BaseCache(object): + """Base Cache Abstraction + + :params parsed_url: Parsed url object. + :params options: A dictionary with configuration parameters + for the cache. For example: + - default_ttl: An integer defining the default ttl + for keys. + """ + + def __init__(self, parsed_url, options=None): + self._parsed_url = parsed_url + self._options = options or {} + self._default_ttl = int(self._options.get('default_ttl', 0)) + + @abc.abstractmethod + def _set(self, key, value, ttl, not_exists=False): + """Implementations of this class have to override this method.""" + + def set(self, key, value, ttl, not_exists=False): + """Sets or updates a cache entry + + NOTE: Thread-safety is required and has to be + guaranteed by the backend implementation. + + :params key: Item key as string. + :type key: `unicode string` + :params value: Value to assign to the key. This + can be anything that is handled + by the current backend. + :params ttl: Key's timeout in seconds. 0 means + no timeout. + :type ttl: int + :params not_exists: If True, the key will be set + if it doesn't exist. Otherwise, + it'll always be set. + :type not_exists: bool + + :returns: True if the operation succeeds, False otherwise. + """ + if ttl is None: + ttl = self._default_ttl + + return self._set(key, value, ttl, not_exists) + + def __setitem__(self, key, value): + self.set(key, value, self._default_ttl) + + def setdefault(self, key, value): + """Sets the key value to `value` if it doesn't exist + + :params key: Item key as string. + :type key: `unicode string` + :params value: Value to assign to the key. This + can be anything that is handled + by the current backend. + """ + try: + return self[key] + except KeyError: + self[key] = value + return value + + @abc.abstractmethod + def _get(self, key, default): + """Implementations of this class have to override this method.""" + + def get(self, key, default=None): + """Gets one item from the cache + + NOTE: Thread-safety is required and it has to be + guaranteed by the backend implementation. + + :params key: Key for the item to retrieve + from the cache. + :params default: The default value to return. + + :returns: `key`'s value in the cache if it exists, + otherwise `default` should be returned. + """ + return self._get(key, default) + + def __getitem__(self, key): + value = self.get(key, NOTSET) + + if value is NOTSET: + raise KeyError + + return value + + @abc.abstractmethod + def __delitem__(self, key): + """Removes an item from cache. + + NOTE: Thread-safety is required and it has to be + guaranteed by the backend implementation. + + :params key: The key to remove. + + :returns: The key value if there's one + """ + + @abc.abstractmethod + def _clear(self): + """Implementations of this class have to override this method.""" + + def clear(self): + """Removes all items from the cache. + + NOTE: Thread-safety is required and it has to be + guaranteed by the backend implementation. + """ + return self._clear() + + @abc.abstractmethod + def _incr(self, key, delta): + """Implementations of this class have to override this method.""" + + def incr(self, key, delta=1): + """Increments the value for a key + + :params key: The key for the value to be incremented + :params delta: Number of units by which to increment + the value. Pass a negative number to + decrement the value. + + :returns: The new value + """ + return self._incr(key, delta) + + @abc.abstractmethod + def _append_tail(self, key, tail): + """Implementations of this class have to override this method.""" + + def append_tail(self, key, tail): + """Appends `tail` to `key`'s value. + + :params key: The key of the value to which + `tail` should be appended. + :params tail: The list of values to append to the + original. + + :returns: The new value + """ + + if not hasattr(tail, "__iter__"): + raise TypeError('Tail must be an iterable') + + if not isinstance(tail, list): + # NOTE(flaper87): Make sure we pass a list + # down to the implementation. Not all drivers + # have support for generators, sets or other + # iterables. + tail = list(tail) + + return self._append_tail(key, tail) + + def append(self, key, value): + """Appends `value` to `key`'s value. + + :params key: The key of the value to which + `tail` should be appended. + :params value: The value to append to the + original. + + :returns: The new value + """ + return self.append_tail(key, [value]) + + @abc.abstractmethod + def __contains__(self, key): + """Verifies that a key exists. + + :params key: The key to verify. + + :returns: True if the key exists, + otherwise False. + """ + + @abc.abstractmethod + def _get_many(self, keys, default): + """Implementations of this class have to override this method.""" + return ((k, self.get(k, default=default)) for k in keys) + + def get_many(self, keys, default=NOTSET): + """Gets keys' value from cache + + :params keys: List of keys to retrieve. + :params default: The default value to return + for each key that is not in + the cache. + + :returns: A generator of (key, value) + """ + return self._get_many(keys, default) + + @abc.abstractmethod + def _set_many(self, data, ttl): + """Implementations of this class have to override this method.""" + + for key, value in data.items(): + self.set(key, value, ttl=ttl) + + def set_many(self, data, ttl=None): + """Puts several items into the cache at once + + Depending on the backend, this operation may or may + not be efficient. The default implementation calls + set for each (key, value) pair passed, other backends + support set_many operations as part of their protocols. + + :params data: A dictionary like {key: val} to store + in the cache. + :params ttl: Key's timeout in seconds. + """ + + if ttl is None: + ttl = self._default_ttl + + self._set_many(data, ttl) + + def update(self, **kwargs): + """Sets several (key, value) paris. + + Refer to the `set_many` docstring. + """ + self.set_many(kwargs, ttl=self._default_ttl) + + @abc.abstractmethod + def _unset_many(self, keys): + """Implementations of this class have to override this method.""" + for key in keys: + del self[key] + + def unset_many(self, keys): + """Removes several keys from the cache at once + + :params keys: List of keys to unset. + """ + self._unset_many(keys) diff --git a/marconi/openstack/common/cache/cache.py b/marconi/openstack/common/cache/cache.py new file mode 100644 index 000000000..297172cac --- /dev/null +++ b/marconi/openstack/common/cache/cache.py @@ -0,0 +1,79 @@ +# Copyright 2013 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Cache library. + +Supported configuration options: + +`default_backend`: Name of the cache backend to use. +`key_namespace`: Namespace under which keys will be created. +""" + +from stevedore import driver + +from marconi.openstack.common.py3kcompat import urlutils + + +def _get_olso_configs(): + """Returns the oslo.config options to register.""" + # NOTE(flaper87): Oslo config should be + # optional. Instead of doing try / except + # at the top of this file, lets import cfg + # here and assume that the caller of this + # function already took care of this dependency. + from oslo.config import cfg + + return [ + cfg.StrOpt('cache_url', default='memory://', + help='Url to connect to the cache backend.') + ] + + +def register_oslo_configs(conf): + """Registers a cache configuration options + + :params conf: Config object. + :type conf: `cfg.ConfigOptions` + """ + conf.register_opts(_get_olso_configs()) + + +def get_cache(url='memory://'): + """Loads the cache backend + + This function loads the cache backend + specified in the given configuration. + + :param conf: Configuration instance to use + """ + + parsed = urlutils.urlparse(url) + backend = parsed.scheme + + query = parsed.query + # NOTE(flaper87): We need the following hack + # for python versions < 2.7.5. Previous versions + # of python parsed query params just for 'known' + # schemes. This was changed in this patch: + # http://hg.python.org/cpython/rev/79e6ff3d9afd + if not query and '?' in parsed.path: + query = parsed.path.split('?', 1)[-1] + parameters = urlutils.parse_qsl(query) + kwargs = {'options': dict(parameters)} + + mgr = driver.DriverManager('marconi.openstack.common.cache.backends', backend, + invoke_on_load=True, + invoke_args=[parsed], + invoke_kwds=kwargs) + return mgr.driver diff --git a/marconi/openstack/common/excutils.py b/marconi/openstack/common/excutils.py index 955cf3ea2..500bc1818 100644 --- a/marconi/openstack/common/excutils.py +++ b/marconi/openstack/common/excutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # Copyright 2012, Red Hat, Inc. # @@ -26,7 +24,7 @@ import traceback import six -from marconi.openstack.common.gettextutils import _ # noqa +from marconi.openstack.common.gettextutils import _ class save_and_reraise_exception(object): @@ -44,13 +42,13 @@ class save_and_reraise_exception(object): In some cases the caller may not want to re-raise the exception, and for those circumstances this context provides a reraise flag that - can be used to suppress the exception. For example: + can be used to suppress the exception. For example:: - except Exception: - with save_and_reraise_exception() as ctxt: - decide_if_need_reraise() - if not should_be_reraised: - ctxt.reraise = False + except Exception: + with save_and_reraise_exception() as ctxt: + decide_if_need_reraise() + if not should_be_reraised: + ctxt.reraise = False """ def __init__(self): self.reraise = True diff --git a/marconi/openstack/common/fileutils.py b/marconi/openstack/common/fileutils.py index 3a8ee4626..715293bed 100644 --- a/marconi/openstack/common/fileutils.py +++ b/marconi/openstack/common/fileutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # @@ -15,13 +13,13 @@ # License for the specific language governing permissions and limitations # under the License. - import contextlib import errno import os +import tempfile from marconi.openstack.common import excutils -from marconi.openstack.common.gettextutils import _ # noqa +from marconi.openstack.common.gettextutils import _ from marconi.openstack.common import log as logging LOG = logging.getLogger(__name__) @@ -109,3 +107,30 @@ def file_open(*args, **kwargs): state at all (for unit tests) """ return file(*args, **kwargs) + + +def write_to_tempfile(content, path=None, suffix='', prefix='tmp'): + """Create temporary file or use existing file. + + This util is needed for creating temporary file with + specified content, suffix and prefix. If path is not None, + it will be used for writing content. If the path doesn't + exist it'll be created. + + :param content: content for temporary file. + :param path: same as parameter 'dir' for mkstemp + :param suffix: same as parameter 'suffix' for mkstemp + :param prefix: same as parameter 'prefix' for mkstemp + + For example: it can be used in database tests for creating + configuration files. + """ + if path: + ensure_tree(path) + + (fd, path) = tempfile.mkstemp(suffix=suffix, dir=path, prefix=prefix) + try: + os.write(fd, content) + finally: + os.close(fd) + return path diff --git a/marconi/openstack/common/gettextutils.py b/marconi/openstack/common/gettextutils.py index 0d3ee561f..5eb86bab3 100644 --- a/marconi/openstack/common/gettextutils.py +++ b/marconi/openstack/common/gettextutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2012 Red Hat, Inc. # Copyright 2013 IBM Corp. # All Rights Reserved. @@ -26,13 +24,10 @@ Usual usage in an openstack.common module: import copy import gettext -import logging +import locale +from logging import handlers import os import re -try: - import UserString as _userString -except ImportError: - import collections as _userString from babel import localedata import six @@ -58,7 +53,7 @@ def enable_lazy(): def _(msg): if USE_LAZY: - return Message(msg, 'marconi') + return Message(msg, domain='marconi') else: if six.PY3: return _t.gettext(msg) @@ -90,11 +85,6 @@ def install(domain, lazy=False): # messages in OpenStack. We override the standard _() function # and % (format string) operation to build Message objects that can # later be translated when we have more information. - # - # Also included below is an example LocaleHandler that translates - # Messages to an associated locale, effectively allowing many logs, - # each with their own locale. - def _lazy_gettext(msg): """Create and return a Message object. @@ -105,7 +95,7 @@ def install(domain, lazy=False): Message encapsulates a string so that we can translate it later when needed. """ - return Message(msg, domain) + return Message(msg, domain=domain) from six import moves moves.builtins.__dict__['_'] = _lazy_gettext @@ -120,182 +110,168 @@ def install(domain, lazy=False): unicode=True) -class Message(_userString.UserString, object): - """Class used to encapsulate translatable messages.""" - def __init__(self, msg, domain): - # _msg is the gettext msgid and should never change - self._msg = msg - self._left_extra_msg = '' - self._right_extra_msg = '' - self._locale = None - self.params = None - self.domain = domain +class Message(six.text_type): + """A Message object is a unicode object that can be translated. - @property - def data(self): - # NOTE(mrodden): this should always resolve to a unicode string - # that best represents the state of the message currently + Translation of Message is done explicitly using the translate() method. + For all non-translation intents and purposes, a Message is simply unicode, + and can be treated as such. + """ - localedir = os.environ.get(self.domain.upper() + '_LOCALEDIR') - if self.locale: - lang = gettext.translation(self.domain, - localedir=localedir, - languages=[self.locale], - fallback=True) - else: - # use system locale for translations - lang = gettext.translation(self.domain, - localedir=localedir, - fallback=True) + def __new__(cls, msgid, msgtext=None, params=None, domain='marconi', *args): + """Create a new Message object. + In order for translation to work gettext requires a message ID, this + msgid will be used as the base unicode text. It is also possible + for the msgid and the base unicode text to be different by passing + the msgtext parameter. + """ + # If the base msgtext is not given, we use the default translation + # of the msgid (which is in English) just in case the system locale is + # not English, so that the base text will be in that locale by default. + if not msgtext: + msgtext = Message._translate_msgid(msgid, domain) + # We want to initialize the parent unicode with the actual object that + # would have been plain unicode if 'Message' was not enabled. + msg = super(Message, cls).__new__(cls, msgtext) + msg.msgid = msgid + msg.domain = domain + msg.params = params + return msg + + def translate(self, desired_locale=None): + """Translate this message to the desired locale. + + :param desired_locale: The desired locale to translate the message to, + if no locale is provided the message will be + translated to the system's default locale. + + :returns: the translated message in unicode + """ + + translated_message = Message._translate_msgid(self.msgid, + self.domain, + desired_locale) + if self.params is None: + # No need for more translation + return translated_message + + # This Message object may have been formatted with one or more + # Message objects as substitution arguments, given either as a single + # argument, part of a tuple, or as one or more values in a dictionary. + # When translating this Message we need to translate those Messages too + translated_params = _translate_args(self.params, desired_locale) + + translated_message = translated_message % translated_params + + return translated_message + + @staticmethod + def _translate_msgid(msgid, domain, desired_locale=None): + if not desired_locale: + system_locale = locale.getdefaultlocale() + # If the system locale is not available to the runtime use English + if not system_locale[0]: + desired_locale = 'en_US' + else: + desired_locale = system_locale[0] + + locale_dir = os.environ.get(domain.upper() + '_LOCALEDIR') + lang = gettext.translation(domain, + localedir=locale_dir, + languages=[desired_locale], + fallback=True) if six.PY3: - ugettext = lang.gettext + translator = lang.gettext else: - ugettext = lang.ugettext + translator = lang.ugettext - full_msg = (self._left_extra_msg + - ugettext(self._msg) + - self._right_extra_msg) + translated_message = translator(msgid) + return translated_message - if self.params is not None: - full_msg = full_msg % self.params + def __mod__(self, other): + # When we mod a Message we want the actual operation to be performed + # by the parent class (i.e. unicode()), the only thing we do here is + # save the original msgid and the parameters in case of a translation + params = self._sanitize_mod_params(other) + unicode_mod = super(Message, self).__mod__(params) + modded = Message(self.msgid, + msgtext=unicode_mod, + params=params, + domain=self.domain) + return modded - return six.text_type(full_msg) + def _sanitize_mod_params(self, other): + """Sanitize the object being modded with this Message. - @property - def locale(self): - return self._locale + - Add support for modding 'None' so translation supports it + - Trim the modded object, which can be a large dictionary, to only + those keys that would actually be used in a translation + - Snapshot the object being modded, in case the message is + translated, it will be used as it was when the Message was created + """ + if other is None: + params = (other,) + elif isinstance(other, dict): + params = self._trim_dictionary_parameters(other) + else: + params = self._copy_param(other) + return params - @locale.setter - def locale(self, value): - self._locale = value - if not self.params: - return + def _trim_dictionary_parameters(self, dict_param): + """Return a dict that only has matching entries in the msgid.""" + # NOTE(luisg): Here we trim down the dictionary passed as parameters + # to avoid carrying a lot of unnecessary weight around in the message + # object, for example if someone passes in Message() % locals() but + # only some params are used, and additionally we prevent errors for + # non-deepcopyable objects by unicoding() them. - # This Message object may have been constructed with one or more - # Message objects as substitution parameters, given as a single - # Message, or a tuple or Map containing some, so when setting the - # locale for this Message we need to set it for those Messages too. - if isinstance(self.params, Message): - self.params.locale = value - return - if isinstance(self.params, tuple): - for param in self.params: - if isinstance(param, Message): - param.locale = value - return - if isinstance(self.params, dict): - for param in self.params.values(): - if isinstance(param, Message): - param.locale = value + # Look for %(param) keys in msgid; + # Skip %% and deal with the case where % is first character on the line + keys = re.findall('(?:[^%]|^)?%\((\w*)\)[a-z]', self.msgid) - def _save_dictionary_parameter(self, dict_param): - full_msg = self.data - # look for %(blah) fields in string; - # ignore %% and deal with the - # case where % is first character on the line - keys = re.findall('(?:[^%]|^)?%\((\w*)\)[a-z]', full_msg) - - # if we don't find any %(blah) blocks but have a %s - if not keys and re.findall('(?:[^%]|^)%[a-z]', full_msg): - # apparently the full dictionary is the parameter - params = copy.deepcopy(dict_param) + # If we don't find any %(param) keys but have a %s + if not keys and re.findall('(?:[^%]|^)%[a-z]', self.msgid): + # Apparently the full dictionary is the parameter + params = self._copy_param(dict_param) else: params = {} + # Save our existing parameters as defaults to protect + # ourselves from losing values if we are called through an + # (erroneous) chain that builds a valid Message with + # arguments, and then does something like "msg % kwds" + # where kwds is an empty dictionary. + src = {} + if isinstance(self.params, dict): + src.update(self.params) + src.update(dict_param) for key in keys: - try: - params[key] = copy.deepcopy(dict_param[key]) - except TypeError: - # cast uncopyable thing to unicode string - params[key] = six.text_type(dict_param[key]) + params[key] = self._copy_param(src[key]) return params - def _save_parameters(self, other): - # we check for None later to see if - # we actually have parameters to inject, - # so encapsulate if our parameter is actually None - if other is None: - self.params = (other, ) - elif isinstance(other, dict): - self.params = self._save_dictionary_parameter(other) - else: - # fallback to casting to unicode, - # this will handle the problematic python code-like - # objects that cannot be deep-copied - try: - self.params = copy.deepcopy(other) - except TypeError: - self.params = six.text_type(other) + def _copy_param(self, param): + try: + return copy.deepcopy(param) + except TypeError: + # Fallback to casting to unicode this will handle the + # python code-like objects that can't be deep-copied + return six.text_type(param) - return self - - # overrides to be more string-like - def __unicode__(self): - return self.data - - def __str__(self): - if six.PY3: - return self.__unicode__() - return self.data.encode('utf-8') - - def __getstate__(self): - to_copy = ['_msg', '_right_extra_msg', '_left_extra_msg', - 'domain', 'params', '_locale'] - new_dict = self.__dict__.fromkeys(to_copy) - for attr in to_copy: - new_dict[attr] = copy.deepcopy(self.__dict__[attr]) - - return new_dict - - def __setstate__(self, state): - for (k, v) in state.items(): - setattr(self, k, v) - - # operator overloads def __add__(self, other): - copied = copy.deepcopy(self) - copied._right_extra_msg += other.__str__() - return copied + msg = _('Message objects do not support addition.') + raise TypeError(msg) def __radd__(self, other): - copied = copy.deepcopy(self) - copied._left_extra_msg += other.__str__() - return copied + return self.__add__(other) - def __mod__(self, other): - # do a format string to catch and raise - # any possible KeyErrors from missing parameters - self.data % other - copied = copy.deepcopy(self) - return copied._save_parameters(other) - - def __mul__(self, other): - return self.data * other - - def __rmul__(self, other): - return other * self.data - - def __getitem__(self, key): - return self.data[key] - - def __getslice__(self, start, end): - return self.data.__getslice__(start, end) - - def __getattribute__(self, name): - # NOTE(mrodden): handle lossy operations that we can't deal with yet - # These override the UserString implementation, since UserString - # uses our __class__ attribute to try and build a new message - # after running the inner data string through the operation. - # At that point, we have lost the gettext message id and can just - # safely resolve to a string instead. - ops = ['capitalize', 'center', 'decode', 'encode', - 'expandtabs', 'ljust', 'lstrip', 'replace', 'rjust', 'rstrip', - 'strip', 'swapcase', 'title', 'translate', 'upper', 'zfill'] - if name in ops: - return getattr(self.data, name) - else: - return _userString.UserString.__getattribute__(self, name) + def __str__(self): + # NOTE(luisg): Logging in python 2.6 tries to str() log records, + # and it expects specifically a UnicodeError in order to proceed. + msg = _('Message objects do not support str() because they may ' + 'contain non-ascii characters. ' + 'Please use unicode() or translate() instead.') + raise UnicodeError(msg) def get_available_languages(domain): @@ -317,49 +293,147 @@ def get_available_languages(domain): # NOTE(luisg): Babel <1.0 used a function called list(), which was # renamed to locale_identifiers() in >=1.0, the requirements master list # requires >=0.9.6, uncapped, so defensively work with both. We can remove - # this check when the master list updates to >=1.0, and all projects udpate + # this check when the master list updates to >=1.0, and update all projects list_identifiers = (getattr(localedata, 'list', None) or getattr(localedata, 'locale_identifiers')) locale_identifiers = list_identifiers() + for i in locale_identifiers: if find(i) is not None: language_list.append(i) + + # NOTE(luisg): Babel>=1.0,<1.3 has a bug where some OpenStack supported + # locales (e.g. 'zh_CN', and 'zh_TW') aren't supported even though they + # are perfectly legitimate locales: + # https://github.com/mitsuhiko/babel/issues/37 + # In Babel 1.3 they fixed the bug and they support these locales, but + # they are still not explicitly "listed" by locale_identifiers(). + # That is why we add the locales here explicitly if necessary so that + # they are listed as supported. + aliases = {'zh': 'zh_CN', + 'zh_Hant_HK': 'zh_HK', + 'zh_Hant': 'zh_TW', + 'fil': 'tl_PH'} + for (locale, alias) in six.iteritems(aliases): + if locale in language_list and alias not in language_list: + language_list.append(alias) + _AVAILABLE_LANGUAGES[domain] = language_list return copy.copy(language_list) -def get_localized_message(message, user_locale): - """Gets a localized version of the given message in the given locale.""" +def translate(obj, desired_locale=None): + """Gets the translated unicode representation of the given object. + + If the object is not translatable it is returned as-is. + If the locale is None the object is translated to the system locale. + + :param obj: the object to translate + :param desired_locale: the locale to translate the message to, if None the + default system locale will be used + :returns: the translated object in unicode, or the original object if + it could not be translated + """ + message = obj + if not isinstance(message, Message): + # If the object to translate is not already translatable, + # let's first get its unicode representation + message = six.text_type(obj) if isinstance(message, Message): - if user_locale: - message.locale = user_locale - return six.text_type(message) - else: - return message + # Even after unicoding() we still need to check if we are + # running with translatable unicode before translating + return message.translate(desired_locale) + return obj -class LocaleHandler(logging.Handler): - """Handler that can have a locale associated to translate Messages. +def _translate_args(args, desired_locale=None): + """Translates all the translatable elements of the given arguments object. - A quick example of how to utilize the Message class above. - LocaleHandler takes a locale and a target logging.Handler object - to forward LogRecord objects to after translating the internal Message. + This method is used for translating the translatable values in method + arguments which include values of tuples or dictionaries. + If the object is not a tuple or a dictionary the object itself is + translated if it is translatable. + + If the locale is None the object is translated to the system locale. + + :param args: the args to translate + :param desired_locale: the locale to translate the args to, if None the + default system locale will be used + :returns: a new args object with the translated contents of the original + """ + if isinstance(args, tuple): + return tuple(translate(v, desired_locale) for v in args) + if isinstance(args, dict): + translated_dict = {} + for (k, v) in six.iteritems(args): + translated_v = translate(v, desired_locale) + translated_dict[k] = translated_v + return translated_dict + return translate(args, desired_locale) + + +class TranslationHandler(handlers.MemoryHandler): + """Handler that translates records before logging them. + + The TranslationHandler takes a locale and a target logging.Handler object + to forward LogRecord objects to after translating them. This handler + depends on Message objects being logged, instead of regular strings. + + The handler can be configured declaratively in the logging.conf as follows: + + [handlers] + keys = translatedlog, translator + + [handler_translatedlog] + class = handlers.WatchedFileHandler + args = ('/var/log/api-localized.log',) + formatter = context + + [handler_translator] + class = openstack.common.log.TranslationHandler + target = translatedlog + args = ('zh_CN',) + + If the specified locale is not available in the system, the handler will + log in the default locale. """ - def __init__(self, locale, target): - """Initialize a LocaleHandler + def __init__(self, locale=None, target=None): + """Initialize a TranslationHandler :param locale: locale to use for translating messages :param target: logging.Handler object to forward LogRecord objects to after translation """ - logging.Handler.__init__(self) + # NOTE(luisg): In order to allow this handler to be a wrapper for + # other handlers, such as a FileHandler, and still be able to + # configure it using logging.conf, this handler has to extend + # MemoryHandler because only the MemoryHandlers' logging.conf + # parsing is implemented such that it accepts a target handler. + handlers.MemoryHandler.__init__(self, capacity=0, target=target) self.locale = locale - self.target = target + + def setFormatter(self, fmt): + self.target.setFormatter(fmt) def emit(self, record): - if isinstance(record.msg, Message): - # set the locale and resolve to a string - record.msg.locale = self.locale + # We save the message from the original record to restore it + # after translation, so other handlers are not affected by this + original_msg = record.msg + original_args = record.args + + try: + self._translate_and_log_record(record) + finally: + record.msg = original_msg + record.args = original_args + + def _translate_and_log_record(self, record): + record.msg = translate(record.msg, self.locale) + + # In addition to translating the message, we also need to translate + # arguments that were passed to the log method that were not part + # of the main message e.g., log.info(_('Some message %s'), this_one)) + record.args = _translate_args(record.args, self.locale) self.target.emit(record) diff --git a/marconi/openstack/common/importutils.py b/marconi/openstack/common/importutils.py index 7a303f93f..4fd9ae2bc 100644 --- a/marconi/openstack/common/importutils.py +++ b/marconi/openstack/common/importutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # diff --git a/marconi/openstack/common/jsonutils.py b/marconi/openstack/common/jsonutils.py index b9646107f..cb59417e0 100644 --- a/marconi/openstack/common/jsonutils.py +++ b/marconi/openstack/common/jsonutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # Copyright 2011 Justin Santa Barbara @@ -41,11 +39,16 @@ import json try: import xmlrpclib except ImportError: - # NOTE(jd): xmlrpclib is not shipped with Python 3 - xmlrpclib = None + # NOTE(jaypipes): xmlrpclib was renamed to xmlrpc.client in Python3 + # however the function and object call signatures + # remained the same. This whole try/except block should + # be removed and replaced with a call to six.moves once + # six 1.4.2 is released. See http://bit.ly/1bqrVzu + import xmlrpc.client as xmlrpclib import six +from marconi.openstack.common import gettextutils from marconi.openstack.common import importutils from marconi.openstack.common import timeutils @@ -123,18 +126,20 @@ def to_primitive(value, convert_instances=False, convert_datetime=True, level=level, max_depth=max_depth) if isinstance(value, dict): - return dict((k, recursive(v)) for k, v in value.iteritems()) + return dict((k, recursive(v)) for k, v in six.iteritems(value)) elif isinstance(value, (list, tuple)): return [recursive(lv) for lv in value] # It's not clear why xmlrpclib created their own DateTime type, but # for our purposes, make it a datetime type which is explicitly # handled - if xmlrpclib and isinstance(value, xmlrpclib.DateTime): + if isinstance(value, xmlrpclib.DateTime): value = datetime.datetime(*tuple(value.timetuple())[:6]) if convert_datetime and isinstance(value, datetime.datetime): return timeutils.strtime(value) + elif isinstance(value, gettextutils.Message): + return value.data elif hasattr(value, 'iteritems'): return recursive(dict(value.iteritems()), level=level + 1) elif hasattr(value, '__iter__'): diff --git a/marconi/openstack/common/local.py b/marconi/openstack/common/local.py index e82f17d0f..0819d5b97 100644 --- a/marconi/openstack/common/local.py +++ b/marconi/openstack/common/local.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # diff --git a/marconi/openstack/common/lockutils.py b/marconi/openstack/common/lockutils.py index 341110b41..594367de8 100644 --- a/marconi/openstack/common/lockutils.py +++ b/marconi/openstack/common/lockutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # @@ -15,7 +13,6 @@ # License for the specific language governing permissions and limitations # under the License. - import contextlib import errno import functools @@ -31,8 +28,7 @@ import weakref from oslo.config import cfg from marconi.openstack.common import fileutils -from marconi.openstack.common.gettextutils import _ # noqa -from marconi.openstack.common import local +from marconi.openstack.common.gettextutils import _ from marconi.openstack.common import log as logging @@ -79,6 +75,12 @@ class _InterProcessLock(object): self.fname = name def __enter__(self): + basedir = os.path.dirname(self.fname) + + if not os.path.exists(basedir): + fileutils.ensure_tree(basedir) + LOG.info(_('Created lock path: %s'), basedir) + self.lockfile = open(self.fname, 'w') while True: @@ -88,6 +90,7 @@ class _InterProcessLock(object): # Also upon reading the MSDN docs for locking(), it seems # to have a laughable 10 attempts "blocking" mechanism. self.trylock() + LOG.debug(_('Got file lock "%s"'), self.fname) return self except IOError as e: if e.errno in (errno.EACCES, errno.EAGAIN): @@ -104,6 +107,7 @@ class _InterProcessLock(object): except IOError: LOG.exception(_("Could not release the acquired lock `%s`"), self.fname) + LOG.debug(_('Released file lock "%s"'), self.fname) def trylock(self): raise NotImplementedError() @@ -139,26 +143,27 @@ _semaphores = weakref.WeakValueDictionary() _semaphores_lock = threading.Lock() -@contextlib.contextmanager -def lock(name, lock_file_prefix=None, external=False, lock_path=None): - """Context based lock +def external_lock(name, lock_file_prefix=None): + with internal_lock(name): + LOG.debug(_('Attempting to grab external lock "%(lock)s"'), + {'lock': name}) - This function yields a `threading.Semaphore` instance (if we don't use - eventlet.monkey_patch(), else `semaphore.Semaphore`) unless external is - True, in which case, it'll yield an InterProcessLock instance. + # NOTE(mikal): the lock name cannot contain directory + # separators + name = name.replace(os.sep, '_') + if lock_file_prefix: + sep = '' if lock_file_prefix.endswith('-') else '-' + name = '%s%s%s' % (lock_file_prefix, sep, name) - :param lock_file_prefix: The lock_file_prefix argument is used to provide - lock files on disk with a meaningful prefix. + if not CONF.lock_path: + raise cfg.RequiredOptError('lock_path') - :param external: The external keyword argument denotes whether this lock - should work across multiple processes. This means that if two different - workers both run a a method decorated with @synchronized('mylock', - external=True), only one of them will execute at a time. + lock_file_path = os.path.join(CONF.lock_path, name) - :param lock_path: The lock_path keyword argument is used to specify a - special location for external lock files to live. If nothing is set, then - CONF.lock_path is used as a default. - """ + return InterProcessLock(lock_file_path) + + +def internal_lock(name): with _semaphores_lock: try: sem = _semaphores[name] @@ -166,58 +171,35 @@ def lock(name, lock_file_prefix=None, external=False, lock_path=None): sem = threading.Semaphore() _semaphores[name] = sem - with sem: - LOG.debug(_('Got semaphore "%(lock)s"'), {'lock': name}) - - # NOTE(mikal): I know this looks odd - if not hasattr(local.strong_store, 'locks_held'): - local.strong_store.locks_held = [] - local.strong_store.locks_held.append(name) - - try: - if external and not CONF.disable_process_locking: - LOG.debug(_('Attempting to grab file lock "%(lock)s"'), - {'lock': name}) - - # We need a copy of lock_path because it is non-local - local_lock_path = lock_path or CONF.lock_path - if not local_lock_path: - raise cfg.RequiredOptError('lock_path') - - if not os.path.exists(local_lock_path): - fileutils.ensure_tree(local_lock_path) - LOG.info(_('Created lock path: %s'), local_lock_path) - - def add_prefix(name, prefix): - if not prefix: - return name - sep = '' if prefix.endswith('-') else '-' - return '%s%s%s' % (prefix, sep, name) - - # NOTE(mikal): the lock name cannot contain directory - # separators - lock_file_name = add_prefix(name.replace(os.sep, '_'), - lock_file_prefix) - - lock_file_path = os.path.join(local_lock_path, lock_file_name) - - try: - lock = InterProcessLock(lock_file_path) - with lock as lock: - LOG.debug(_('Got file lock "%(lock)s" at %(path)s'), - {'lock': name, 'path': lock_file_path}) - yield lock - finally: - LOG.debug(_('Released file lock "%(lock)s" at %(path)s'), - {'lock': name, 'path': lock_file_path}) - else: - yield sem - - finally: - local.strong_store.locks_held.remove(name) + LOG.debug(_('Got semaphore "%(lock)s"'), {'lock': name}) + return sem -def synchronized(name, lock_file_prefix=None, external=False, lock_path=None): +@contextlib.contextmanager +def lock(name, lock_file_prefix=None, external=False): + """Context based lock + + This function yields a `threading.Semaphore` instance (if we don't use + eventlet.monkey_patch(), else `semaphore.Semaphore`) unless external is + True, in which case, it'll yield an InterProcessLock instance. + + :param lock_file_prefix: The lock_file_prefix argument is used to provide + lock files on disk with a meaningful prefix. + + :param external: The external keyword argument denotes whether this lock + should work across multiple processes. This means that if two different + workers both run a a method decorated with @synchronized('mylock', + external=True), only one of them will execute at a time. + """ + if external and not CONF.disable_process_locking: + lock = external_lock(name, lock_file_prefix) + else: + lock = internal_lock(name) + with lock: + yield lock + + +def synchronized(name, lock_file_prefix=None, external=False): """Synchronization decorator. Decorating a method like so:: @@ -245,7 +227,7 @@ def synchronized(name, lock_file_prefix=None, external=False, lock_path=None): @functools.wraps(f) def inner(*args, **kwargs): try: - with lock(name, lock_file_prefix, external, lock_path): + with lock(name, lock_file_prefix, external): LOG.debug(_('Got semaphore / lock "%(function)s"'), {'function': f.__name__}) return f(*args, **kwargs) diff --git a/marconi/openstack/common/log.py b/marconi/openstack/common/log.py index e2c752f5d..7aa637e85 100644 --- a/marconi/openstack/common/log.py +++ b/marconi/openstack/common/log.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. @@ -35,13 +33,15 @@ import logging import logging.config import logging.handlers import os +import re import sys import traceback from oslo.config import cfg +import six from six import moves -from marconi.openstack.common.gettextutils import _ # noqa +from marconi.openstack.common.gettextutils import _ from marconi.openstack.common import importutils from marconi.openstack.common import jsonutils from marconi.openstack.common import local @@ -49,6 +49,24 @@ from marconi.openstack.common import local _DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" +_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password'] + +# NOTE(ldbragst): Let's build a list of regex objects using the list of +# _SANITIZE_KEYS we already have. This way, we only have to add the new key +# to the list of _SANITIZE_KEYS and we can generate regular expressions +# for XML and JSON automatically. +_SANITIZE_PATTERNS = [] +_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])', + r'(<%(key)s>).*?()', + r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])', + r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])'] + +for key in _SANITIZE_KEYS: + for pattern in _FORMAT_PATTERNS: + reg_ex = re.compile(pattern % {'key': key}, re.DOTALL) + _SANITIZE_PATTERNS.append(reg_ex) + + common_cli_opts = [ cfg.BoolOpt('debug', short='d', @@ -63,11 +81,13 @@ common_cli_opts = [ ] logging_cli_opts = [ - cfg.StrOpt('log-config', + cfg.StrOpt('log-config-append', metavar='PATH', - help='If this option is specified, the logging configuration ' - 'file specified is used and overrides any other logging ' - 'options specified. Please see the Python logging module ' + deprecated_name='log-config', + help='The name of logging configuration file. It does not ' + 'disable existing loggers, but just appends specified ' + 'logging configuration to any other existing logging ' + 'options. Please see the Python logging module ' 'documentation for details on logging configuration ' 'files.'), cfg.StrOpt('log-format', @@ -110,7 +130,7 @@ generic_log_opts = [ log_opts = [ cfg.StrOpt('logging_context_format_string', default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s ' - '%(name)s [%(request_id)s %(user)s %(tenant)s] ' + '%(name)s [%(request_id)s %(user_identity)s] ' '%(instance)s%(message)s', help='format string to use for log messages with context'), cfg.StrOpt('logging_default_format_string', @@ -126,12 +146,13 @@ log_opts = [ help='prefix each line of exception output with this format'), cfg.ListOpt('default_log_levels', default=[ + 'amqp=WARN', 'amqplib=WARN', - 'sqlalchemy=WARN', 'boto=WARN', + 'qpid=WARN', + 'sqlalchemy=WARN', 'suds=INFO', - 'keystone=INFO', - 'eventlet.wsgi.server=WARN' + 'iso8601=WARN', ], help='list of logger=LEVEL pairs'), cfg.BoolOpt('publish_errors', @@ -207,6 +228,42 @@ def _get_log_file_path(binary=None): binary = binary or _get_binary_name() return '%s.log' % (os.path.join(logdir, binary),) + return None + + +def mask_password(message, secret="***"): + """Replace password with 'secret' in message. + + :param message: The string which includes security information. + :param secret: value with which to replace passwords. + :returns: The unicode value of message with the password fields masked. + + For example: + + >>> mask_password("'adminPass' : 'aaaaa'") + "'adminPass' : '***'" + >>> mask_password("'admin_pass' : 'aaaaa'") + "'admin_pass' : '***'" + >>> mask_password('"password" : "aaaaa"') + '"password" : "***"' + >>> mask_password("'original_password' : 'aaaaa'") + "'original_password' : '***'" + >>> mask_password("u'original_password' : u'aaaaa'") + "u'original_password' : u'***'" + """ + message = six.text_type(message) + + # NOTE(ldbragst): Check to see if anything in message contains any key + # specified in _SANITIZE_KEYS, if not then just return the message since + # we don't have to mask any passwords. + if not any(key in message for key in _SANITIZE_KEYS): + return message + + secret = r'\g<1>' + secret + r'\g<2>' + for pattern in _SANITIZE_PATTERNS: + message = re.sub(pattern, secret, message) + return message + class BaseLoggerAdapter(logging.LoggerAdapter): @@ -249,6 +306,13 @@ class ContextAdapter(BaseLoggerAdapter): self.warn(stdmsg, *args, **kwargs) def process(self, msg, kwargs): + # NOTE(mrodden): catch any Message/other object and + # coerce to unicode before they can get + # to the python logging and possibly + # cause string encoding trouble + if not isinstance(msg, six.string_types): + msg = six.text_type(msg) + if 'extra' not in kwargs: kwargs['extra'] = {} extra = kwargs['extra'] @@ -268,10 +332,12 @@ class ContextAdapter(BaseLoggerAdapter): elif instance_uuid: instance_extra = (CONF.instance_uuid_format % {'uuid': instance_uuid}) - extra.update({'instance': instance_extra}) + extra['instance'] = instance_extra - extra.update({"project": self.project}) - extra.update({"version": self.version}) + extra.setdefault('user_identity', kwargs.pop('user_identity', None)) + + extra['project'] = self.project + extra['version'] = self.version extra['extra'] = extra.copy() return msg, kwargs @@ -285,7 +351,7 @@ class JSONFormatter(logging.Formatter): def formatException(self, ei, strip_newlines=True): lines = traceback.format_exception(*ei) if strip_newlines: - lines = [itertools.ifilter( + lines = [moves.filter( lambda x: x, line.rstrip().splitlines()) for line in lines] lines = list(itertools.chain(*lines)) @@ -323,11 +389,13 @@ class JSONFormatter(logging.Formatter): def _create_logging_excepthook(product_name): - def logging_excepthook(type, value, tb): + def logging_excepthook(exc_type, value, tb): extra = {} - if CONF.verbose: - extra['exc_info'] = (type, value, tb) - getLogger(product_name).critical(str(value), **extra) + if CONF.verbose or CONF.debug: + extra['exc_info'] = (exc_type, value, tb) + getLogger(product_name).critical( + "".join(traceback.format_exception_only(exc_type, value)), + **extra) return logging_excepthook @@ -344,17 +412,18 @@ class LogConfigError(Exception): err_msg=self.err_msg) -def _load_log_config(log_config): +def _load_log_config(log_config_append): try: - logging.config.fileConfig(log_config) + logging.config.fileConfig(log_config_append, + disable_existing_loggers=False) except moves.configparser.Error as exc: - raise LogConfigError(log_config, str(exc)) + raise LogConfigError(log_config_append, str(exc)) def setup(product_name): """Setup logging.""" - if CONF.log_config: - _load_log_config(CONF.log_config) + if CONF.log_config_append: + _load_log_config(CONF.log_config_append) else: _setup_logging_from_conf() sys.excepthook = _create_logging_excepthook(product_name) @@ -410,7 +479,7 @@ def _setup_logging_from_conf(): streamlog = ColorHandler() log_root.addHandler(streamlog) - elif not CONF.log_file: + elif not logpath: # pass sys.stdout as a positional argument # python2.6 calls the argument strm, in 2.7 it's stream streamlog = logging.StreamHandler(sys.stdout) @@ -494,7 +563,7 @@ class ContextFormatter(logging.Formatter): def format(self, record): """Uses contextstring if request_id is set, otherwise default.""" - # NOTE(sdague): default the fancier formating params + # NOTE(sdague): default the fancier formatting params # to an empty string so we don't throw an exception if # they get used for key in ('instance', 'color'): @@ -510,7 +579,7 @@ class ContextFormatter(logging.Formatter): CONF.logging_debug_format_suffix): self._fmt += " " + CONF.logging_debug_format_suffix - # Cache this on the record, Logger will respect our formated copy + # Cache this on the record, Logger will respect our formatted copy if record.exc_info: record.exc_text = self.formatException(record.exc_info, record) return logging.Formatter.format(self, record) diff --git a/marconi/openstack/common/py3kcompat/__init__.py b/marconi/openstack/common/py3kcompat/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/marconi/openstack/common/py3kcompat/urlutils.py b/marconi/openstack/common/py3kcompat/urlutils.py new file mode 100644 index 000000000..84e457a44 --- /dev/null +++ b/marconi/openstack/common/py3kcompat/urlutils.py @@ -0,0 +1,67 @@ +# +# Copyright 2013 Canonical Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +""" +Python2/Python3 compatibility layer for OpenStack +""" + +import six + +if six.PY3: + # python3 + import urllib.error + import urllib.parse + import urllib.request + + urlencode = urllib.parse.urlencode + urljoin = urllib.parse.urljoin + quote = urllib.parse.quote + quote_plus = urllib.parse.quote_plus + parse_qsl = urllib.parse.parse_qsl + unquote = urllib.parse.unquote + unquote_plus = urllib.parse.unquote_plus + urlparse = urllib.parse.urlparse + urlsplit = urllib.parse.urlsplit + urlunsplit = urllib.parse.urlunsplit + SplitResult = urllib.parse.SplitResult + + urlopen = urllib.request.urlopen + URLError = urllib.error.URLError + pathname2url = urllib.request.pathname2url +else: + # python2 + import urllib + import urllib2 + import urlparse + + urlencode = urllib.urlencode + quote = urllib.quote + quote_plus = urllib.quote_plus + unquote = urllib.unquote + unquote_plus = urllib.unquote_plus + + parse = urlparse + parse_qsl = parse.parse_qsl + urljoin = parse.urljoin + urlparse = parse.urlparse + urlsplit = parse.urlsplit + urlunsplit = parse.urlunsplit + SplitResult = parse.SplitResult + + urlopen = urllib2.urlopen + URLError = urllib2.URLError + pathname2url = urllib.pathname2url diff --git a/marconi/openstack/common/timeutils.py b/marconi/openstack/common/timeutils.py index 98d877d59..52688a026 100644 --- a/marconi/openstack/common/timeutils.py +++ b/marconi/openstack/common/timeutils.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2011 OpenStack Foundation. # All Rights Reserved. # @@ -50,9 +48,9 @@ def parse_isotime(timestr): try: return iso8601.parse_date(timestr) except iso8601.ParseError as e: - raise ValueError(unicode(e)) + raise ValueError(six.text_type(e)) except TypeError as e: - raise ValueError(unicode(e)) + raise ValueError(six.text_type(e)) def strtime(at=None, fmt=PERFECT_TIME_FORMAT): @@ -79,6 +77,9 @@ def is_older_than(before, seconds): """Return True if before is older than seconds.""" if isinstance(before, six.string_types): before = parse_strtime(before).replace(tzinfo=None) + else: + before = before.replace(tzinfo=None) + return utcnow() - before > datetime.timedelta(seconds=seconds) @@ -86,6 +87,9 @@ def is_newer_than(after, seconds): """Return True if after is newer than seconds.""" if isinstance(after, six.string_types): after = parse_strtime(after).replace(tzinfo=None) + else: + after = after.replace(tzinfo=None) + return after - utcnow() > datetime.timedelta(seconds=seconds) @@ -110,7 +114,7 @@ def utcnow(): def iso8601_from_timestamp(timestamp): - """Returns a iso8601 formated date from timestamp.""" + """Returns a iso8601 formatted date from timestamp.""" return isotime(datetime.datetime.utcfromtimestamp(timestamp)) @@ -178,6 +182,15 @@ def delta_seconds(before, after): datetime objects (as a float, to microsecond resolution). """ delta = after - before + return total_seconds(delta) + + +def total_seconds(delta): + """Return the total seconds of datetime.timedelta object. + + Compute total seconds of datetime.timedelta, datetime.timedelta + doesn't have method total_seconds in Python2.6, calculate it manually. + """ try: return delta.total_seconds() except AttributeError: @@ -188,8 +201,8 @@ def delta_seconds(before, after): def is_soon(dt, window): """Determines if time is going to happen in the next window seconds. - :params dt: the time - :params window: minimum seconds to remain to consider the time not soon + :param dt: the time + :param window: minimum seconds to remain to consider the time not soon :return: True if expiration is within the given duration """ diff --git a/marconi/queues/bootstrap.py b/marconi/queues/bootstrap.py index aef0a6e7a..605c665f7 100644 --- a/marconi/queues/bootstrap.py +++ b/marconi/queues/bootstrap.py @@ -16,9 +16,9 @@ from oslo.config import cfg from stevedore import driver -from marconi.common.cache import cache as oslo_cache from marconi.common import decorators from marconi.common import errors +from marconi.openstack.common.cache import cache as oslo_cache from marconi.openstack.common.gettextutils import _ from marconi.openstack.common import log from marconi.queues.storage import pipeline @@ -90,7 +90,8 @@ class Bootstrap(object): def cache(self): LOG.debug(_(u'Loading proxy cache driver')) try: - mgr = oslo_cache.get_cache(self.conf) + oslo_cache.register_oslo_configs(self.conf) + mgr = oslo_cache.get_cache(self.conf.cache_url) return mgr except RuntimeError as exc: LOG.exception(exc) diff --git a/marconi/tests/queues/storage/base.py b/marconi/tests/queues/storage/base.py index 37a8c3cd7..5b5d778e7 100644 --- a/marconi/tests/queues/storage/base.py +++ b/marconi/tests/queues/storage/base.py @@ -21,7 +21,7 @@ import ddt import six from testtools import matchers -from marconi.common.cache import cache as oslo_cache +from marconi.openstack.common.cache import cache as oslo_cache from marconi.openstack.common import timeutils from marconi.queues import storage from marconi.queues.storage import errors @@ -47,7 +47,8 @@ class ControllerBaseTest(testing.TestBase): self.controller_class, self.controller_base_class)) - cache = oslo_cache.get_cache(self.conf) + oslo_cache.register_oslo_configs(self.conf) + cache = oslo_cache.get_cache(self.conf.cache_url) self.driver = self.driver_class(self.conf, cache) self._prepare_conf() diff --git a/openstack-common.conf b/openstack-common.conf index d5813d723..f13751052 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -1,4 +1,5 @@ [DEFAULT] +module=cache module=excutils module=fileutils module=gettextutils diff --git a/setup.cfg b/setup.cfg index 7e10dd0f9..c481dbf64 100644 --- a/setup.cfg +++ b/setup.cfg @@ -45,9 +45,8 @@ marconi.queues.public.transport = marconi.queues.admin.transport = wsgi = marconi.queues.transport.wsgi.admin.driver:Driver -marconi.common.cache.backends = - memory = marconi.common.cache._backends.memory:MemoryBackend - memcached = marconi.common.cache._backends.memcached:MemcachedBackend +marconi.openstack.common.cache.backends = + memory = marconi.openstack.common.cache._backends.memory:MemoryBackend [nosetests] where=tests diff --git a/tests/unit/queues/storage/test_impl_mongodb.py b/tests/unit/queues/storage/test_impl_mongodb.py index fb310c7b0..2bcffc8bf 100644 --- a/tests/unit/queues/storage/test_impl_mongodb.py +++ b/tests/unit/queues/storage/test_impl_mongodb.py @@ -21,7 +21,7 @@ from pymongo import cursor import pymongo.errors from testtools import matchers -from marconi.common.cache import cache as oslo_cache +from marconi.openstack.common.cache import cache as oslo_cache from marconi.openstack.common import timeutils from marconi.queues import storage from marconi.queues.storage import errors diff --git a/tests/unit/queues/storage/test_shard_catalog.py b/tests/unit/queues/storage/test_shard_catalog.py index f30eb6ef4..5133077fd 100644 --- a/tests/unit/queues/storage/test_shard_catalog.py +++ b/tests/unit/queues/storage/test_shard_catalog.py @@ -18,7 +18,7 @@ import uuid from oslo.config import cfg -from marconi.common.cache import cache as oslo_cache +from marconi.openstack.common.cache import cache as oslo_cache from marconi.queues.storage import sharding from marconi.queues.storage import sqlite from marconi.queues.storage import utils diff --git a/tests/unit/queues/storage/test_shard_queues.py b/tests/unit/queues/storage/test_shard_queues.py index f5a6aa485..9957379e5 100644 --- a/tests/unit/queues/storage/test_shard_queues.py +++ b/tests/unit/queues/storage/test_shard_queues.py @@ -19,7 +19,7 @@ import uuid from oslo.config import cfg -from marconi.common.cache import cache as oslo_cache +from marconi.openstack.common.cache import cache as oslo_cache from marconi.queues.storage import sharding from marconi.queues.storage import utils from marconi import tests as testing