diff --git a/tooz/drivers/memcached.py b/tooz/drivers/memcached.py index e7e2bfc6..414e7f4d 100644 --- a/tooz/drivers/memcached.py +++ b/tooz/drivers/memcached.py @@ -134,20 +134,22 @@ class MemcachedDriver(coordination.CoordinationDriver): def __init__(self, member_id, parsed_url, options): super(MemcachedDriver, self).__init__() + options = utils.collapse(options) + self._options = options self._member_id = member_id self._groups = set() self._executor = None self.host = (parsed_url.hostname or "localhost", parsed_url.port or 11211) - default_timeout = options.get('timeout', ['30']) - self.timeout = int(default_timeout[-1]) + default_timeout = options.get('timeout', '30') + self.timeout = int(default_timeout) self.membership_timeout = int(options.get( - 'membership_timeout', default_timeout)[-1]) + 'membership_timeout', default_timeout)) self.lock_timeout = int(options.get( - 'lock_timeout', default_timeout)[-1]) + 'lock_timeout', default_timeout)) self.leader_timeout = int(options.get( - 'leader_timeout', default_timeout)[-1]) - max_pool_size = options.get('max_pool_size', [None])[-1] + 'leader_timeout', default_timeout)) + max_pool_size = options.get('max_pool_size', None) if max_pool_size is not None: self.max_pool_size = int(max_pool_size) else: diff --git a/tooz/drivers/mysql.py b/tooz/drivers/mysql.py index fa91fbc9..d87c0a65 100644 --- a/tooz/drivers/mysql.py +++ b/tooz/drivers/mysql.py @@ -101,7 +101,7 @@ class MySQLDriver(coordination.CoordinationDriver): """Initialize the MySQL driver.""" super(MySQLDriver, self).__init__() self._parsed_url = parsed_url - self._options = options + self._options = utils.collapse(options) def _start(self): self._conn = MySQLDriver.get_connection(self._parsed_url, @@ -144,7 +144,7 @@ class MySQLDriver(coordination.CoordinationDriver): dbname = parsed_url.path[1:] username = parsed_url.username password = parsed_url.password - unix_socket = options.get("unix_socket", [None])[-1] + unix_socket = options.get("unix_socket") try: if unix_socket: diff --git a/tooz/drivers/pgsql.py b/tooz/drivers/pgsql.py index b5abca5b..02624cac 100644 --- a/tooz/drivers/pgsql.py +++ b/tooz/drivers/pgsql.py @@ -160,7 +160,7 @@ class PostgresDriver(coordination.CoordinationDriver): """Initialize the PostgreSQL driver.""" super(PostgresDriver, self).__init__() self._parsed_url = parsed_url - self._options = options + self._options = utils.collapse(options) def _start(self): self._conn = PostgresDriver.get_connection(self._parsed_url, @@ -198,9 +198,9 @@ class PostgresDriver(coordination.CoordinationDriver): @staticmethod def get_connection(parsed_url, options): - host = options.get("host", [None])[-1] - port = parsed_url.port or options.get("port", [None])[-1] - dbname = parsed_url.path[1:] or options.get("dbname", [None])[-1] + host = options.get("host") + port = parsed_url.port or options.get("port") + dbname = parsed_url.path[1:] or options.get("dbname") username = parsed_url.username password = parsed_url.password diff --git a/tooz/drivers/redis.py b/tooz/drivers/redis.py index 6e773aa9..1766f252 100644 --- a/tooz/drivers/redis.py +++ b/tooz/drivers/redis.py @@ -242,17 +242,17 @@ class RedisDriver(coordination.CoordinationDriver): def __init__(self, member_id, parsed_url, options): super(RedisDriver, self).__init__() + options = utils.collapse(options, exclude=self._CLIENT_LIST_ARGS) self._parsed_url = parsed_url self._options = options - encoding = options.get('encoding', [self._DEFAULT_ENCODING]) - self._encoding = encoding[-1] - timeout = options.get('timeout', [self._CLIENT_DEFAULT_SOCKET_TO]) - self.timeout = int(timeout[-1]) + self._encoding = options.get('encoding', self._DEFAULT_ENCODING) + timeout = options.get('timeout', self._CLIENT_DEFAULT_SOCKET_TO) + self.timeout = int(timeout) self.membership_timeout = float(options.get( - 'membership_timeout', timeout)[-1]) - lock_timeout = options.get('lock_timeout', [self.timeout]) - self.lock_timeout = int(lock_timeout[-1]) - namespace = options.get('namespace', ['_tooz'])[-1] + 'membership_timeout', timeout)) + lock_timeout = options.get('lock_timeout', self.timeout) + self.lock_timeout = int(lock_timeout) + namespace = options.get('namespace', '_tooz') self._namespace = self._to_binary(namespace) self._group_prefix = self._namespace + b"_group" self._leader_prefix = self._namespace + b"_leader" @@ -322,22 +322,14 @@ class RedisDriver(coordination.CoordinationDriver): for a in cls._CLIENT_ARGS: if a not in options: continue - # The reason the last index is used is that when multiple options - # of the same name are given via a url the values will be - # accumulated in a list (and not just be a single value)... - # - # For ex: the following is a valid url which will have 2 values - # for the 'timeout' argument: - # - # redis://localhost:6379?timeout=5&timeout=2 if a in cls._CLIENT_BOOL_ARGS: - v = strutils.bool_from_string(options[a][-1]) + v = strutils.bool_from_string(options[a]) elif a in cls._CLIENT_LIST_ARGS: v = options[a] elif a in cls._CLIENT_INT_ARGS: - v = int(options[a][-1]) + v = int(options[a]) else: - v = options[a][-1] + v = options[a] kwargs[a] = v if 'socket_timeout' not in kwargs: kwargs['socket_timeout'] = default_socket_timeout diff --git a/tooz/drivers/zake.py b/tooz/drivers/zake.py index 5c19e814..87c69dd9 100644 --- a/tooz/drivers/zake.py +++ b/tooz/drivers/zake.py @@ -42,7 +42,7 @@ class ZakeDriver(zookeeper.KazooDriver): @classmethod def _make_client(cls, parsed_url, options): if 'storage' in options: - storage = options['storage'][-1] + storage = options['storage'] else: storage = cls.fake_storage return fake_client.FakeClient(storage=storage) diff --git a/tooz/drivers/zookeeper.py b/tooz/drivers/zookeeper.py index acfec6c5..fb1bc33d 100644 --- a/tooz/drivers/zookeeper.py +++ b/tooz/drivers/zookeeper.py @@ -73,10 +73,11 @@ class BaseZooKeeperDriver(coordination.CoordinationDriver): def __init__(self, member_id, parsed_url, options): super(BaseZooKeeperDriver, self).__init__() + options = utils.collapse(options) + self._options = options self._member_id = member_id - self.timeout = int(options.get('timeout', ['10'])[-1]) - namespace = options.get('namespace', [self._TOOZ_NAMESPACE]) - self._namespace = namespace[-1] + self.timeout = int(options.get('timeout', '10')) + self._namespace = options.get('namespace', self._TOOZ_NAMESPACE) def _start(self): try: @@ -340,7 +341,7 @@ class KazooDriver(BaseZooKeeperDriver): def __init__(self, member_id, parsed_url, options): super(KazooDriver, self).__init__(member_id, parsed_url, options) - self._coord = self._make_client(parsed_url, options) + self._coord = self._make_client(parsed_url, self._options) self._member_id = member_id self._timeout_exception = self._coord.handler.timeout_exception diff --git a/tooz/tests/test_utils.py b/tooz/tests/test_utils.py new file mode 100644 index 00000000..c61a73f3 --- /dev/null +++ b/tooz/tests/test_utils.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2015 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import six +from testtools import testcase + +from tooz import utils + + +class TestUtilsCollapse(testcase.TestCase): + + def test_bad_type(self): + self.assertRaises(TypeError, utils.collapse, "") + self.assertRaises(TypeError, utils.collapse, []) + self.assertRaises(TypeError, utils.collapse, 2) + + def test_collapse_simple(self): + ex = { + 'a': [1], + 'b': 2, + 'c': (1, 2, 3), + } + c_ex = utils.collapse(ex) + self.assertEqual({'a': 1, 'c': 3, 'b': 2}, c_ex) + + def test_collapse_exclusions(self): + ex = { + 'a': [1], + 'b': 2, + 'c': (1, 2, 3), + } + c_ex = utils.collapse(ex, exclude=['a']) + self.assertEqual({'a': [1], 'c': 3, 'b': 2}, c_ex) + + def test_no_collapse(self): + ex = { + 'a': [1], + 'b': [2], + 'c': (1, 2, 3), + } + c_ex = utils.collapse(ex, exclude=set(six.iterkeys(ex))) + self.assertEqual(ex, c_ex) + + def test_custom_selector(self): + ex = { + 'a': [1, 2, 3], + } + c_ex = utils.collapse(ex, + item_selector=lambda items: items[0]) + self.assertEqual({'a': 1}, c_ex) + + def test_empty_lists(self): + ex = { + 'a': [], + 'b': (), + 'c': [1], + } + c_ex = utils.collapse(ex) + self.assertNotIn('b', c_ex) + self.assertNotIn('a', c_ex) + self.assertIn('c', c_ex) diff --git a/tooz/utils.py b/tooz/utils.py index b6b716be..c1619d90 100644 --- a/tooz/utils.py +++ b/tooz/utils.py @@ -21,6 +21,35 @@ import six from tooz import coordination +def collapse(config, exclude=None, item_selector=None): + """Collapses config with keys and **list/tuple** values. + + NOTE(harlowja): The last item/index from the list/tuple value is selected + be default as the new value (values that are not lists/tuples are left + alone). If the list/tuple value is empty (zero length), then no value + is set. + """ + if not isinstance(config, dict): + raise TypeError("Unexpected config type, dict expected") + if not config: + return {} + if exclude is None: + exclude = set() + if item_selector is None: + item_selector = lambda items: items[-1] + collapsed = {} + for (k, v) in six.iteritems(config): + if isinstance(v, (tuple, list)): + if k in exclude: + collapsed[k] = v + else: + if len(v): + collapsed[k] = item_selector(v) + else: + collapsed[k] = v + return collapsed + + def exception_message(exc): """Return the string representation of exception.""" try: