openstack
/
tooz
Code Issues Proposed changes

Provide and use a options collapsing function 

Instead of having each driver index the last value of
each option, when all current drivers do not actually care
about any of the other indexes we can just use and provide
a option collapsing function that we by default apply to
all the existing drivers options (to avoid needing to index
by -1).

This avoids a confusion point for new folks, and one that
does not currently really provide any value (no driver in
tree besides one option in the redis driver uses/consumes
or understands multiple options in the first place).

Change-Id: Ia2898d84fd0e54871b829f4a95786a33accc20b8
This commit is contained in:
Joshua Harlow 2015-05-01 15:21:09 -07:00
parent 9c329cc535
commit cdac135a57
8 changed files with 136 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
tooz/drivers/memcached.py
View File

@ -134,20 +134,22 @@ class MemcachedDriver(coordination.CoordinationDriver):
def __init__(self, member_id, parsed_url, options):
super(MemcachedDriver, self).__init__()
options = utils.collapse(options)
self._options = options
self._member_id = member_id
self._groups = set()
self._executor = None
self.host = (parsed_url.hostname or "localhost",
parsed_url.port or 11211)
default_timeout = options.get('timeout', ['30'])
self.timeout = int(default_timeout[-1])
default_timeout = options.get('timeout', '30')
self.timeout = int(default_timeout)
self.membership_timeout = int(options.get(
'membership_timeout', default_timeout)[-1])
'membership_timeout', default_timeout))
self.lock_timeout = int(options.get(
'lock_timeout', default_timeout)[-1])
'lock_timeout', default_timeout))
self.leader_timeout = int(options.get(
'leader_timeout', default_timeout)[-1])
max_pool_size = options.get('max_pool_size', [None])[-1]
'leader_timeout', default_timeout))
max_pool_size = options.get('max_pool_size', None)
if max_pool_size is not None:
self.max_pool_size = int(max_pool_size)
else:

4
tooz/drivers/mysql.py
View File

@ -101,7 +101,7 @@ class MySQLDriver(coordination.CoordinationDriver):
"""Initialize the MySQL driver."""
super(MySQLDriver, self).__init__()
self._parsed_url = parsed_url
self._options = options
self._options = utils.collapse(options)
def _start(self):
self._conn = MySQLDriver.get_connection(self._parsed_url,
@ -144,7 +144,7 @@ class MySQLDriver(coordination.CoordinationDriver):
dbname = parsed_url.path[1:]
username = parsed_url.username
password = parsed_url.password
unix_socket = options.get("unix_socket", [None])[-1]
unix_socket = options.get("unix_socket")
try:
if unix_socket:

8
tooz/drivers/pgsql.py
View File

@ -160,7 +160,7 @@ class PostgresDriver(coordination.CoordinationDriver):
"""Initialize the PostgreSQL driver."""
super(PostgresDriver, self).__init__()
self._parsed_url = parsed_url
self._options = options
self._options = utils.collapse(options)
def _start(self):
self._conn = PostgresDriver.get_connection(self._parsed_url,
@ -198,9 +198,9 @@ class PostgresDriver(coordination.CoordinationDriver):
@staticmethod
def get_connection(parsed_url, options):
host = options.get("host", [None])[-1]
port = parsed_url.port or options.get("port", [None])[-1]
dbname = parsed_url.path[1:] or options.get("dbname", [None])[-1]
host = options.get("host")
port = parsed_url.port or options.get("port")
dbname = parsed_url.path[1:] or options.get("dbname")
username = parsed_url.username
password = parsed_url.password

30
tooz/drivers/redis.py
View File

@ -242,17 +242,17 @@ class RedisDriver(coordination.CoordinationDriver):
def __init__(self, member_id, parsed_url, options):
super(RedisDriver, self).__init__()
options = utils.collapse(options, exclude=self._CLIENT_LIST_ARGS)
self._parsed_url = parsed_url
self._options = options
encoding = options.get('encoding', [self._DEFAULT_ENCODING])
self._encoding = encoding[-1]
timeout = options.get('timeout', [self._CLIENT_DEFAULT_SOCKET_TO])
self.timeout = int(timeout[-1])
self._encoding = options.get('encoding', self._DEFAULT_ENCODING)
timeout = options.get('timeout', self._CLIENT_DEFAULT_SOCKET_TO)
self.timeout = int(timeout)
self.membership_timeout = float(options.get(
'membership_timeout', timeout)[-1])
lock_timeout = options.get('lock_timeout', [self.timeout])
self.lock_timeout = int(lock_timeout[-1])
namespace = options.get('namespace', ['_tooz'])[-1]
'membership_timeout', timeout))
lock_timeout = options.get('lock_timeout', self.timeout)
self.lock_timeout = int(lock_timeout)
namespace = options.get('namespace', '_tooz')
self._namespace = self._to_binary(namespace)
self._group_prefix = self._namespace + b"_group"
self._leader_prefix = self._namespace + b"_leader"
@ -322,22 +322,14 @@ class RedisDriver(coordination.CoordinationDriver):
for a in cls._CLIENT_ARGS:
if a not in options:
continue
# The reason the last index is used is that when multiple options
# of the same name are given via a url the values will be
# accumulated in a list (and not just be a single value)...
#
# For ex: the following is a valid url which will have 2 values
# for the 'timeout' argument:
#
# redis://localhost:6379?timeout=5&timeout=2
if a in cls._CLIENT_BOOL_ARGS:
v = strutils.bool_from_string(options[a][-1])
v = strutils.bool_from_string(options[a])
elif a in cls._CLIENT_LIST_ARGS:
v = options[a]
elif a in cls._CLIENT_INT_ARGS:
v = int(options[a][-1])
v = int(options[a])
else:
v = options[a][-1]
v = options[a]
kwargs[a] = v
if 'socket_timeout' not in kwargs:
kwargs['socket_timeout'] = default_socket_timeout

2
tooz/drivers/zake.py
View File

@ -42,7 +42,7 @@ class ZakeDriver(zookeeper.KazooDriver):
@classmethod
def _make_client(cls, parsed_url, options):
if 'storage' in options:
storage = options['storage'][-1]
storage = options['storage']
else:
storage = cls.fake_storage
return fake_client.FakeClient(storage=storage)

9
tooz/drivers/zookeeper.py
View File

@ -73,10 +73,11 @@ class BaseZooKeeperDriver(coordination.CoordinationDriver):
def __init__(self, member_id, parsed_url, options):
super(BaseZooKeeperDriver, self).__init__()
options = utils.collapse(options)
self._options = options
self._member_id = member_id
self.timeout = int(options.get('timeout', ['10'])[-1])
namespace = options.get('namespace', [self._TOOZ_NAMESPACE])
self._namespace = namespace[-1]
self.timeout = int(options.get('timeout', '10'))
self._namespace = options.get('namespace', self._TOOZ_NAMESPACE)
def _start(self):
try:
@ -340,7 +341,7 @@ class KazooDriver(BaseZooKeeperDriver):
def __init__(self, member_id, parsed_url, options):
super(KazooDriver, self).__init__(member_id, parsed_url, options)
self._coord = self._make_client(parsed_url, options)
self._coord = self._make_client(parsed_url, self._options)
self._member_id = member_id
self._timeout_exception = self._coord.handler.timeout_exception

76
tooz/tests/test_utils.py Normal file
View File

@ -0,0 +1,76 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2015 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import six
from testtools import testcase
from tooz import utils
class TestUtilsCollapse(testcase.TestCase):
def test_bad_type(self):
self.assertRaises(TypeError, utils.collapse, "")
self.assertRaises(TypeError, utils.collapse, [])
self.assertRaises(TypeError, utils.collapse, 2)
def test_collapse_simple(self):
ex = {
'a': [1],
'b': 2,
'c': (1, 2, 3),
}
c_ex = utils.collapse(ex)
self.assertEqual({'a': 1, 'c': 3, 'b': 2}, c_ex)
def test_collapse_exclusions(self):
ex = {
'a': [1],
'b': 2,
'c': (1, 2, 3),
}
c_ex = utils.collapse(ex, exclude=['a'])
self.assertEqual({'a': [1], 'c': 3, 'b': 2}, c_ex)
def test_no_collapse(self):
ex = {
'a': [1],
'b': [2],
'c': (1, 2, 3),
}
c_ex = utils.collapse(ex, exclude=set(six.iterkeys(ex)))
self.assertEqual(ex, c_ex)
def test_custom_selector(self):
ex = {
'a': [1, 2, 3],
}
c_ex = utils.collapse(ex,
item_selector=lambda items: items[0])
self.assertEqual({'a': 1}, c_ex)
def test_empty_lists(self):
ex = {
'a': [],
'b': (),
'c': [1],
}
c_ex = utils.collapse(ex)
self.assertNotIn('b', c_ex)
self.assertNotIn('a', c_ex)
self.assertIn('c', c_ex)

29
tooz/utils.py
View File

@ -21,6 +21,35 @@ import six
from tooz import coordination
def collapse(config, exclude=None, item_selector=None):
"""Collapses config with keys and **list/tuple** values.
NOTE(harlowja): The last item/index from the list/tuple value is selected
be default as the new value (values that are not lists/tuples are left
alone). If the list/tuple value is empty (zero length), then no value
is set.
"""
if not isinstance(config, dict):
raise TypeError("Unexpected config type, dict expected")
if not config:
return {}
if exclude is None:
exclude = set()
if item_selector is None:
item_selector = lambda items: items[-1]
collapsed = {}
for (k, v) in six.iteritems(config):
if isinstance(v, (tuple, list)):
if k in exclude:
collapsed[k] = v
else:
if len(v):
collapsed[k] = item_selector(v)
else:
collapsed[k] = v
return collapsed
def exception_message(exc):
"""Return the string representation of exception."""
try: