etcd: driver with lock support
Change-Id: Ibac90b9b2a751eb4f502e2f8b723e5608dcaad18
This commit is contained in:
parent
c08caaef07
commit
d2529173ec
|
@ -15,3 +15,6 @@ AUTHORS
|
||||||
ChangeLog
|
ChangeLog
|
||||||
# Generated by testrepository
|
# Generated by testrepository
|
||||||
.testrepository
|
.testrepository
|
||||||
|
# Generated by etcd
|
||||||
|
etcd-v*
|
||||||
|
default.etcd
|
||||||
|
|
|
@ -24,11 +24,11 @@ APIs
|
||||||
Driver support
|
Driver support
|
||||||
--------------
|
--------------
|
||||||
|
|
||||||
=========================================== ========================================= ===================================================== ============================================= ================================================ ============================================= =========================================== =================================================
|
=========================================== =========================================== ========================================= ===================================================== ============================================= ================================================ ============================================= =========================================== =================================================
|
||||||
:py:class:`~tooz.drivers.file.FileDriver` :py:class:`~tooz.drivers.ipc.IPCDriver` :py:class:`~tooz.drivers.memcached.MemcachedDriver` :py:class:`~tooz.drivers.mysql.MySQLDriver` :py:class:`~tooz.drivers.pgsql.PostgresDriver` :py:class:`~tooz.drivers.redis.RedisDriver` :py:class:`~tooz.drivers.zake.ZakeDriver` :py:class:`~tooz.drivers.zookeeper.KazooDriver`
|
:py:class:`~tooz.drivers.etcd.EtcdDriver` :py:class:`~tooz.drivers.file.FileDriver` :py:class:`~tooz.drivers.ipc.IPCDriver` :py:class:`~tooz.drivers.memcached.MemcachedDriver` :py:class:`~tooz.drivers.mysql.MySQLDriver` :py:class:`~tooz.drivers.pgsql.PostgresDriver` :py:class:`~tooz.drivers.redis.RedisDriver` :py:class:`~tooz.drivers.zake.ZakeDriver` :py:class:`~tooz.drivers.zookeeper.KazooDriver`
|
||||||
=========================================== ========================================= ===================================================== ============================================= ================================================ ============================================= =========================================== =================================================
|
=========================================== =========================================== ========================================= ===================================================== ============================================= ================================================ ============================================= =========================================== =================================================
|
||||||
Yes No Yes No No Yes Yes Yes
|
No Yes No Yes No No Yes Yes Yes
|
||||||
=========================================== ========================================= ===================================================== ============================================= ================================================ ============================================= =========================================== =================================================
|
=========================================== =========================================== ========================================= ===================================================== ============================================= ================================================ ============================================= =========================================== =================================================
|
||||||
|
|
||||||
Leaders
|
Leaders
|
||||||
=======
|
=======
|
||||||
|
@ -44,11 +44,11 @@ APIs
|
||||||
Driver support
|
Driver support
|
||||||
--------------
|
--------------
|
||||||
|
|
||||||
=========================================== ========================================= ===================================================== ============================================= ================================================ ============================================= =========================================== =================================================
|
=========================================== =========================================== ========================================= ===================================================== ============================================= ================================================ ============================================= =========================================== =================================================
|
||||||
:py:class:`~tooz.drivers.file.FileDriver` :py:class:`~tooz.drivers.ipc.IPCDriver` :py:class:`~tooz.drivers.memcached.MemcachedDriver` :py:class:`~tooz.drivers.mysql.MySQLDriver` :py:class:`~tooz.drivers.pgsql.PostgresDriver` :py:class:`~tooz.drivers.redis.RedisDriver` :py:class:`~tooz.drivers.zake.ZakeDriver` :py:class:`~tooz.drivers.zookeeper.KazooDriver`
|
:py:class:`~tooz.drivers.etcd.EtcdDriver` :py:class:`~tooz.drivers.file.FileDriver` :py:class:`~tooz.drivers.ipc.IPCDriver` :py:class:`~tooz.drivers.memcached.MemcachedDriver` :py:class:`~tooz.drivers.mysql.MySQLDriver` :py:class:`~tooz.drivers.pgsql.PostgresDriver` :py:class:`~tooz.drivers.redis.RedisDriver` :py:class:`~tooz.drivers.zake.ZakeDriver` :py:class:`~tooz.drivers.zookeeper.KazooDriver`
|
||||||
=========================================== ========================================= ===================================================== ============================================= ================================================ ============================================= =========================================== =================================================
|
=========================================== =========================================== ========================================= ===================================================== ============================================= ================================================ ============================================= =========================================== =================================================
|
||||||
No No Yes No No Yes Yes Yes
|
No No No Yes No No Yes Yes Yes
|
||||||
=========================================== ========================================= ===================================================== ============================================= ================================================ ============================================= =========================================== =================================================
|
=========================================== =========================================== ========================================= ===================================================== ============================================= ================================================ ============================================= =========================================== =================================================
|
||||||
|
|
||||||
Locking
|
Locking
|
||||||
=======
|
=======
|
||||||
|
@ -61,8 +61,9 @@ APIs
|
||||||
Driver support
|
Driver support
|
||||||
--------------
|
--------------
|
||||||
|
|
||||||
=========================================== ========================================= ===================================================== ============================================= ================================================ ============================================= =========================================== =================================================
|
=========================================== =========================================== ========================================= ===================================================== ============================================= ================================================ ============================================= =========================================== =================================================
|
||||||
:py:class:`~tooz.drivers.file.FileDriver` :py:class:`~tooz.drivers.ipc.IPCDriver` :py:class:`~tooz.drivers.memcached.MemcachedDriver` :py:class:`~tooz.drivers.mysql.MySQLDriver` :py:class:`~tooz.drivers.pgsql.PostgresDriver` :py:class:`~tooz.drivers.redis.RedisDriver` :py:class:`~tooz.drivers.zake.ZakeDriver` :py:class:`~tooz.drivers.zookeeper.KazooDriver`
|
:py:class:`~tooz.drivers.etcd.EtcdDriver` :py:class:`~tooz.drivers.file.FileDriver` :py:class:`~tooz.drivers.ipc.IPCDriver` :py:class:`~tooz.drivers.memcached.MemcachedDriver` :py:class:`~tooz.drivers.mysql.MySQLDriver` :py:class:`~tooz.drivers.pgsql.PostgresDriver` :py:class:`~tooz.drivers.redis.RedisDriver` :py:class:`~tooz.drivers.zake.ZakeDriver` :py:class:`~tooz.drivers.zookeeper.KazooDriver`
|
||||||
=========================================== ========================================= ===================================================== ============================================= ================================================ ============================================= =========================================== =================================================
|
=========================================== =========================================== ========================================= ===================================================== ============================================= ================================================ ============================================= =========================================== =================================================
|
||||||
Yes Yes Yes Yes Yes Yes Yes Yes
|
Yes Yes Yes Yes Yes Yes Yes Yes Yes
|
||||||
=========================================== ========================================= ===================================================== ============================================= ================================================ ============================================= =========================================== =================================================
|
=========================================== =========================================== ========================================= ===================================================== ============================================= ================================================ ============================================= =========================================== =================================================
|
||||||
|
|
||||||
|
|
|
@ -8,6 +8,12 @@ Interfaces
|
||||||
.. autoclass:: tooz.coordination.CoordinationDriver
|
.. autoclass:: tooz.coordination.CoordinationDriver
|
||||||
:members:
|
:members:
|
||||||
|
|
||||||
|
Etcd
|
||||||
|
~~~
|
||||||
|
|
||||||
|
.. autoclass:: tooz.drivers.etcd.EtcdDriver
|
||||||
|
:members:
|
||||||
|
|
||||||
File
|
File
|
||||||
~~~~
|
~~~~
|
||||||
|
|
||||||
|
|
|
@ -14,3 +14,4 @@ futures>=3.0;python_version=='2.7' or python_version=='2.6'
|
||||||
futurist>=0.1.2 # Apache-2.0
|
futurist>=0.1.2 # Apache-2.0
|
||||||
oslo.utils>=3.2.0 # Apache-2.0
|
oslo.utils>=3.2.0 # Apache-2.0
|
||||||
oslo.serialization>=1.10.0 # Apache-2.0
|
oslo.serialization>=1.10.0 # Apache-2.0
|
||||||
|
requests!=2.9.0,>=2.8.1
|
||||||
|
|
|
@ -0,0 +1,44 @@
|
||||||
|
#!/bin/bash
|
||||||
|
set -eux
|
||||||
|
|
||||||
|
clean_exit() {
|
||||||
|
local error_code="$?"
|
||||||
|
kill $(jobs -p)
|
||||||
|
return $error_code
|
||||||
|
}
|
||||||
|
|
||||||
|
trap clean_exit EXIT
|
||||||
|
if [ -n "$(which etcd)" ]; then
|
||||||
|
etcd &
|
||||||
|
else
|
||||||
|
ETCD_VERSION=2.2.2
|
||||||
|
case `uname -s` in
|
||||||
|
Darwin)
|
||||||
|
OS=darwin
|
||||||
|
SUFFIX=zip
|
||||||
|
;;
|
||||||
|
Linux)
|
||||||
|
OS=linux
|
||||||
|
SUFFIX=tar.gz
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Unsupported OS"
|
||||||
|
exit 1
|
||||||
|
esac
|
||||||
|
case `uname -m` in
|
||||||
|
x86_64)
|
||||||
|
MACHINE=amd64
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Unsupported machine"
|
||||||
|
exit 1
|
||||||
|
esac
|
||||||
|
TARBALL_NAME=etcd-v${ETCD_VERSION}-$OS-$MACHINE
|
||||||
|
test ! -d "$TARBALL_NAME" && curl -L https://github.com/coreos/etcd/releases/download/v${ETCD_VERSION}/${TARBALL_NAME}.${SUFFIX} | tar xz
|
||||||
|
cd "$TARBALL_NAME"
|
||||||
|
./etcd &
|
||||||
|
fi
|
||||||
|
|
||||||
|
export TOOZ_TEST_ETCD_URL="etcd://localhost:4001"
|
||||||
|
# Yield execution to venv command
|
||||||
|
$*
|
|
@ -25,6 +25,7 @@ packages =
|
||||||
|
|
||||||
[entry_points]
|
[entry_points]
|
||||||
tooz.backends =
|
tooz.backends =
|
||||||
|
etcd = tooz.drivers.etcd:EtcdDriver
|
||||||
kazoo = tooz.drivers.zookeeper:KazooDriver
|
kazoo = tooz.drivers.zookeeper:KazooDriver
|
||||||
zake = tooz.drivers.zake:ZakeDriver
|
zake = tooz.drivers.zake:ZakeDriver
|
||||||
memcached = tooz.drivers.memcached:MemcachedDriver
|
memcached = tooz.drivers.memcached:MemcachedDriver
|
||||||
|
|
|
@ -33,6 +33,7 @@ def print_methods(methods):
|
||||||
|
|
||||||
driver_tpl = ":py:class:`~tooz.drivers.%s`"
|
driver_tpl = ":py:class:`~tooz.drivers.%s`"
|
||||||
driver_class_names = [
|
driver_class_names = [
|
||||||
|
"etcd.EtcdDriver",
|
||||||
"file.FileDriver",
|
"file.FileDriver",
|
||||||
"ipc.IPCDriver",
|
"ipc.IPCDriver",
|
||||||
"memcached.MemcachedDriver",
|
"memcached.MemcachedDriver",
|
||||||
|
@ -71,6 +72,7 @@ print_header("Driver support", delim="-")
|
||||||
print("")
|
print("")
|
||||||
grouping_table = [
|
grouping_table = [
|
||||||
[
|
[
|
||||||
|
"No", # Etcd
|
||||||
"Yes", # File
|
"Yes", # File
|
||||||
"No", # IPC
|
"No", # IPC
|
||||||
"Yes", # Memcached
|
"Yes", # Memcached
|
||||||
|
@ -101,6 +103,7 @@ print_header("Driver support", delim="-")
|
||||||
print("")
|
print("")
|
||||||
leader_table = [
|
leader_table = [
|
||||||
[
|
[
|
||||||
|
"No", # Etcd
|
||||||
"No", # File
|
"No", # File
|
||||||
"No", # IPC
|
"No", # IPC
|
||||||
"Yes", # Memcached
|
"Yes", # Memcached
|
||||||
|
@ -128,6 +131,7 @@ print_header("Driver support", delim="-")
|
||||||
print("")
|
print("")
|
||||||
lock_table = [
|
lock_table = [
|
||||||
[
|
[
|
||||||
|
"Yes", # Etcd
|
||||||
"Yes", # File
|
"Yes", # File
|
||||||
"Yes", # IPC
|
"Yes", # IPC
|
||||||
"Yes", # Memcached
|
"Yes", # Memcached
|
||||||
|
|
|
@ -0,0 +1,201 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from oslo_utils import timeutils
|
||||||
|
import requests
|
||||||
|
import six
|
||||||
|
|
||||||
|
import tooz
|
||||||
|
from tooz import coordination
|
||||||
|
from tooz import locking
|
||||||
|
from tooz import utils
|
||||||
|
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _translate_failures(func):
|
||||||
|
"""Translates common requests exceptions into tooz exceptions."""
|
||||||
|
|
||||||
|
@six.wraps(func)
|
||||||
|
def wrapper(*args, **kwargs):
|
||||||
|
try:
|
||||||
|
return func(*args, **kwargs)
|
||||||
|
except requests.exceptions.RequestException as e:
|
||||||
|
coordination.raise_with_cause(coordination.ToozConnectionError,
|
||||||
|
utils.exception_message(e),
|
||||||
|
cause=e)
|
||||||
|
|
||||||
|
return wrapper
|
||||||
|
|
||||||
|
|
||||||
|
class _Client(object):
|
||||||
|
def __init__(self, host, port, protocol):
|
||||||
|
self.host = host
|
||||||
|
self.port = port
|
||||||
|
self.protocol = protocol
|
||||||
|
self.session = requests.Session()
|
||||||
|
|
||||||
|
@property
|
||||||
|
def base_url(self):
|
||||||
|
return self.protocol + '://' + self.host + ':' + str(self.port)
|
||||||
|
|
||||||
|
def get_url(self, path):
|
||||||
|
return self.base_url + '/v2/' + path.lstrip("/")
|
||||||
|
|
||||||
|
def get(self, url, **kwargs):
|
||||||
|
return self.session.get(self.get_url(url), **kwargs).json()
|
||||||
|
|
||||||
|
def put(self, url, **kwargs):
|
||||||
|
return self.session.put(self.get_url(url), **kwargs).json()
|
||||||
|
|
||||||
|
def delete(self, url, **kwargs):
|
||||||
|
return self.session.delete(self.get_url(url), **kwargs).json()
|
||||||
|
|
||||||
|
def self_stats(self):
|
||||||
|
return self.session.get(self.get_url("/stats/self"))
|
||||||
|
|
||||||
|
|
||||||
|
class EtcdLock(locking.Lock):
|
||||||
|
_TOOZ_LOCK_PREFIX = "tooz_locks"
|
||||||
|
|
||||||
|
def __init__(self, name, coord, client, ttl=60):
|
||||||
|
super(EtcdLock, self).__init__(name)
|
||||||
|
self.client = client
|
||||||
|
self.coord = coord
|
||||||
|
self.lock = None
|
||||||
|
self.ttl = ttl
|
||||||
|
# NOTE(jd) sha1 of the name to be sure it works with any string?
|
||||||
|
self._lock_url = (
|
||||||
|
"/keys/" + self._TOOZ_LOCK_PREFIX + "/" + name.decode('ascii')
|
||||||
|
)
|
||||||
|
|
||||||
|
def acquire(self, blocking=True):
|
||||||
|
if isinstance(blocking, bool):
|
||||||
|
watch = None
|
||||||
|
else:
|
||||||
|
watch = timeutils.StopWatch(duration=blocking)
|
||||||
|
watch.start()
|
||||||
|
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
reply = self.client.put(
|
||||||
|
self._lock_url,
|
||||||
|
timeout=watch.leftover() if watch else None,
|
||||||
|
data={"ttl": self.ttl,
|
||||||
|
"prevExist": "false"})
|
||||||
|
except requests.exceptions.RequestException:
|
||||||
|
if watch and watch.leftover() == 0:
|
||||||
|
return False
|
||||||
|
|
||||||
|
# We got the lock!
|
||||||
|
if reply.get("errorCode") is None:
|
||||||
|
self.coord._acquired_locks.append(self)
|
||||||
|
return True
|
||||||
|
|
||||||
|
# We didn't get the lock and we don't want to wait
|
||||||
|
if blocking is False:
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Ok, so let's wait a bit (or forever!)
|
||||||
|
try:
|
||||||
|
reply = self.client.get(
|
||||||
|
self._lock_url
|
||||||
|
+ "?wait=true&waitIndex=%d" % reply['index'],
|
||||||
|
timeout=watch.leftover() if watch else None)
|
||||||
|
except requests.exceptions.RequestException:
|
||||||
|
if watch and watch.expired():
|
||||||
|
return False
|
||||||
|
|
||||||
|
@_translate_failures
|
||||||
|
def release(self):
|
||||||
|
if self in self.coord._acquired_locks:
|
||||||
|
reply = self.client.delete(self._lock_url)
|
||||||
|
if reply.get("errorCode") is None:
|
||||||
|
self.coord._acquired_locks.remove(self)
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
@_translate_failures
|
||||||
|
def heartbeat(self):
|
||||||
|
"""Keep the lock alive."""
|
||||||
|
poked = self.client.put(self._lock_url,
|
||||||
|
data={"ttl": self.ttl,
|
||||||
|
"prevExist": "true"})
|
||||||
|
errorcode = poked.get("errorCode")
|
||||||
|
if errorcode:
|
||||||
|
LOG.warn("Unable to heartbeat by updating key '%s' with extended"
|
||||||
|
" expiry of %s seconds: %d, %s", self.name, self.ttl,
|
||||||
|
errorcode, poked.get("message"))
|
||||||
|
|
||||||
|
|
||||||
|
class EtcdDriver(coordination.CoordinationDriver):
|
||||||
|
"""An etcd based driver.
|
||||||
|
|
||||||
|
This driver uses etcd provide the coordination driver semantics and
|
||||||
|
required API(s).
|
||||||
|
"""
|
||||||
|
|
||||||
|
#: Default socket/lock/member/leader timeout used when none is provided.
|
||||||
|
DEFAULT_TIMEOUT = 30
|
||||||
|
|
||||||
|
def __init__(self, member_id, parsed_url, options):
|
||||||
|
super(EtcdDriver, self).__init__()
|
||||||
|
options = utils.collapse(options)
|
||||||
|
self.client = _Client(host=parsed_url.hostname,
|
||||||
|
port=parsed_url.port,
|
||||||
|
protocol=options.get('protocol', 'http'))
|
||||||
|
default_timeout = options.get('timeout', self.DEFAULT_TIMEOUT)
|
||||||
|
self.lock_timeout = int(options.get(
|
||||||
|
'lock_timeout', default_timeout))
|
||||||
|
self._acquired_locks = []
|
||||||
|
|
||||||
|
def _start(self):
|
||||||
|
try:
|
||||||
|
self.client.self_stats()
|
||||||
|
except requests.exceptions.ConnectionError as e:
|
||||||
|
raise coordination.ToozConnectionError(utils.exception_message(e))
|
||||||
|
|
||||||
|
def get_lock(self, name):
|
||||||
|
return EtcdLock(name, self, self.client, self.lock_timeout)
|
||||||
|
|
||||||
|
def heartbeat(self):
|
||||||
|
for lock in self._acquired_locks:
|
||||||
|
lock.heartbeat()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def watch_join_group(group_id, callback):
|
||||||
|
raise tooz.NotImplemented
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def unwatch_join_group(group_id, callback):
|
||||||
|
raise tooz.NotImplemented
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def watch_leave_group(group_id, callback):
|
||||||
|
raise tooz.NotImplemented
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def unwatch_leave_group(group_id, callback):
|
||||||
|
raise tooz.NotImplemented
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def watch_elected_as_leader(group_id, callback):
|
||||||
|
raise tooz.NotImplemented
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def unwatch_elected_as_leader(group_id, callback):
|
||||||
|
raise tooz.NotImplemented
|
|
@ -60,6 +60,8 @@ class TestAPI(testscenarios.TestWithScenarios,
|
||||||
'bad_url': 'mysql://localhost:1'}),
|
'bad_url': 'mysql://localhost:1'}),
|
||||||
('zookeeper', {'url': os.getenv("TOOZ_TEST_ZOOKEEPER_URL"),
|
('zookeeper', {'url': os.getenv("TOOZ_TEST_ZOOKEEPER_URL"),
|
||||||
'bad_url': 'zookeeper://localhost:1'}),
|
'bad_url': 'zookeeper://localhost:1'}),
|
||||||
|
('etcd', {'url': os.getenv("TOOZ_TEST_ETCD_URL"),
|
||||||
|
'bad_url': 'etcd://localhost:1'})
|
||||||
]
|
]
|
||||||
|
|
||||||
def assertRaisesAny(self, exc_classes, callable_obj, *args, **kwargs):
|
def assertRaisesAny(self, exc_classes, callable_obj, *args, **kwargs):
|
||||||
|
|
7
tox.ini
7
tox.ini
|
@ -56,6 +56,13 @@ commands = {toxinidir}/setup-mysql-env.sh python setup.py testr --slowest --test
|
||||||
basepython = python3.4
|
basepython = python3.4
|
||||||
commands = {toxinidir}/setup-mysql-env.sh python setup.py testr --slowest --testr-args="{posargs}"
|
commands = {toxinidir}/setup-mysql-env.sh python setup.py testr --slowest --testr-args="{posargs}"
|
||||||
|
|
||||||
|
[testenv:py27-etcd]
|
||||||
|
commands = {toxinidir}/setup-etcd-env.sh python setup.py testr --slowest --testr-args="{posargs}"
|
||||||
|
|
||||||
|
[testenv:py34-etcd]
|
||||||
|
basepython = python3.4
|
||||||
|
commands = {toxinidir}/setup-etcd-env.sh python setup.py testr --slowest --testr-args="{posargs}"
|
||||||
|
|
||||||
[testenv:cover]
|
[testenv:cover]
|
||||||
commands = python setup.py testr --slowest --coverage --testr-args="{posargs}"
|
commands = python setup.py testr --slowest --coverage --testr-args="{posargs}"
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue