etcd: driver with lock support

Change-Id: Ibac90b9b2a751eb4f502e2f8b723e5608dcaad18

.gitignore

@@ -15,3 +15,6 @@ AUTHORS
 ChangeLog
 # Generated by testrepository
 .testrepository
+# Generated by etcd
+etcd-v*
+default.etcd

doc/source/compatibility.rst

@@ -24,11 +24,11 @@ APIs
 Driver support
 --------------
 
-===========================================  =========================================  =====================================================  =============================================  ================================================  =============================================  ===========================================  =================================================
-:py:class:`~tooz.drivers.file.FileDriver`    :py:class:`~tooz.drivers.ipc.IPCDriver`    :py:class:`~tooz.drivers.memcached.MemcachedDriver`    :py:class:`~tooz.drivers.mysql.MySQLDriver`    :py:class:`~tooz.drivers.pgsql.PostgresDriver`    :py:class:`~tooz.drivers.redis.RedisDriver`    :py:class:`~tooz.drivers.zake.ZakeDriver`    :py:class:`~tooz.drivers.zookeeper.KazooDriver`
-===========================================  =========================================  =====================================================  =============================================  ================================================  =============================================  ===========================================  =================================================
-Yes                                          No                                         Yes                                                    No                                             No                                                Yes                                            Yes                                          Yes
-===========================================  =========================================  =====================================================  =============================================  ================================================  =============================================  ===========================================  =================================================
+===========================================  ===========================================  =========================================  =====================================================  =============================================  ================================================  =============================================  ===========================================  =================================================
+:py:class:`~tooz.drivers.etcd.EtcdDriver`    :py:class:`~tooz.drivers.file.FileDriver`    :py:class:`~tooz.drivers.ipc.IPCDriver`    :py:class:`~tooz.drivers.memcached.MemcachedDriver`    :py:class:`~tooz.drivers.mysql.MySQLDriver`    :py:class:`~tooz.drivers.pgsql.PostgresDriver`    :py:class:`~tooz.drivers.redis.RedisDriver`    :py:class:`~tooz.drivers.zake.ZakeDriver`    :py:class:`~tooz.drivers.zookeeper.KazooDriver`
+===========================================  ===========================================  =========================================  =====================================================  =============================================  ================================================  =============================================  ===========================================  =================================================
+No                                           Yes                                          No                                         Yes                                                    No                                             No                                                Yes                                            Yes                                          Yes
+===========================================  ===========================================  =========================================  =====================================================  =============================================  ================================================  =============================================  ===========================================  =================================================
 
 Leaders
 =======
@@ -44,11 +44,11 @@ APIs
 Driver support
 --------------
 
-===========================================  =========================================  =====================================================  =============================================  ================================================  =============================================  ===========================================  =================================================
-:py:class:`~tooz.drivers.file.FileDriver`    :py:class:`~tooz.drivers.ipc.IPCDriver`    :py:class:`~tooz.drivers.memcached.MemcachedDriver`    :py:class:`~tooz.drivers.mysql.MySQLDriver`    :py:class:`~tooz.drivers.pgsql.PostgresDriver`    :py:class:`~tooz.drivers.redis.RedisDriver`    :py:class:`~tooz.drivers.zake.ZakeDriver`    :py:class:`~tooz.drivers.zookeeper.KazooDriver`
-===========================================  =========================================  =====================================================  =============================================  ================================================  =============================================  ===========================================  =================================================
-No                                           No                                         Yes                                                    No                                             No                                                Yes                                            Yes                                          Yes
-===========================================  =========================================  =====================================================  =============================================  ================================================  =============================================  ===========================================  =================================================
+===========================================  ===========================================  =========================================  =====================================================  =============================================  ================================================  =============================================  ===========================================  =================================================
+:py:class:`~tooz.drivers.etcd.EtcdDriver`    :py:class:`~tooz.drivers.file.FileDriver`    :py:class:`~tooz.drivers.ipc.IPCDriver`    :py:class:`~tooz.drivers.memcached.MemcachedDriver`    :py:class:`~tooz.drivers.mysql.MySQLDriver`    :py:class:`~tooz.drivers.pgsql.PostgresDriver`    :py:class:`~tooz.drivers.redis.RedisDriver`    :py:class:`~tooz.drivers.zake.ZakeDriver`    :py:class:`~tooz.drivers.zookeeper.KazooDriver`
+===========================================  ===========================================  =========================================  =====================================================  =============================================  ================================================  =============================================  ===========================================  =================================================
+No                                           No                                           No                                         Yes                                                    No                                             No                                                Yes                                            Yes                                          Yes
+===========================================  ===========================================  =========================================  =====================================================  =============================================  ================================================  =============================================  ===========================================  =================================================
 
 Locking
 =======
@@ -61,8 +61,9 @@ APIs
 Driver support
 --------------
 
-===========================================  =========================================  =====================================================  =============================================  ================================================  =============================================  ===========================================  =================================================
-:py:class:`~tooz.drivers.file.FileDriver`    :py:class:`~tooz.drivers.ipc.IPCDriver`    :py:class:`~tooz.drivers.memcached.MemcachedDriver`    :py:class:`~tooz.drivers.mysql.MySQLDriver`    :py:class:`~tooz.drivers.pgsql.PostgresDriver`    :py:class:`~tooz.drivers.redis.RedisDriver`    :py:class:`~tooz.drivers.zake.ZakeDriver`    :py:class:`~tooz.drivers.zookeeper.KazooDriver`
-===========================================  =========================================  =====================================================  =============================================  ================================================  =============================================  ===========================================  =================================================
-Yes                                          Yes                                        Yes                                                    Yes                                            Yes                                               Yes                                            Yes                                          Yes
-===========================================  =========================================  =====================================================  =============================================  ================================================  =============================================  ===========================================  =================================================
+===========================================  ===========================================  =========================================  =====================================================  =============================================  ================================================  =============================================  ===========================================  =================================================
+:py:class:`~tooz.drivers.etcd.EtcdDriver`    :py:class:`~tooz.drivers.file.FileDriver`    :py:class:`~tooz.drivers.ipc.IPCDriver`    :py:class:`~tooz.drivers.memcached.MemcachedDriver`    :py:class:`~tooz.drivers.mysql.MySQLDriver`    :py:class:`~tooz.drivers.pgsql.PostgresDriver`    :py:class:`~tooz.drivers.redis.RedisDriver`    :py:class:`~tooz.drivers.zake.ZakeDriver`    :py:class:`~tooz.drivers.zookeeper.KazooDriver`
+===========================================  ===========================================  =========================================  =====================================================  =============================================  ================================================  =============================================  ===========================================  =================================================
+Yes                                          Yes                                          Yes                                        Yes                                                    Yes                                            Yes                                               Yes                                            Yes                                          Yes
+===========================================  ===========================================  =========================================  =====================================================  =============================================  ================================================  =============================================  ===========================================  =================================================
+

doc/source/developers.rst

@@ -8,6 +8,12 @@ Interfaces
 .. autoclass:: tooz.coordination.CoordinationDriver
    :members:
 
+Etcd
+~~~
+
+.. autoclass:: tooz.drivers.etcd.EtcdDriver
+   :members:
+
 File
 ~~~~
 

requirements.txt

@@ -14,3 +14,4 @@ futures>=3.0;python_version=='2.7' or python_version=='2.6'
 futurist>=0.1.2 # Apache-2.0
 oslo.utils>=3.2.0 # Apache-2.0
 oslo.serialization>=1.10.0 # Apache-2.0
+requests!=2.9.0,>=2.8.1

setup-etcd-env.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+set -eux
+
+clean_exit() {
+    local error_code="$?"
+    kill $(jobs -p)
+    return $error_code
+}
+
+trap clean_exit EXIT
+if [ -n "$(which etcd)" ]; then
+    etcd &
+else
+    ETCD_VERSION=2.2.2
+    case `uname -s` in
+        Darwin)
+            OS=darwin
+            SUFFIX=zip
+            ;;
+        Linux)
+            OS=linux
+            SUFFIX=tar.gz
+            ;;
+        *)
+            echo "Unsupported OS"
+            exit 1
+    esac
+    case `uname -m` in
+         x86_64)
+             MACHINE=amd64
+             ;;
+         *)
+            echo "Unsupported machine"
+            exit 1
+    esac
+    TARBALL_NAME=etcd-v${ETCD_VERSION}-$OS-$MACHINE
+    test ! -d "$TARBALL_NAME" && curl -L https://github.com/coreos/etcd/releases/download/v${ETCD_VERSION}/${TARBALL_NAME}.${SUFFIX} | tar xz
+    cd "$TARBALL_NAME"
+    ./etcd &
+fi
+
+export TOOZ_TEST_ETCD_URL="etcd://localhost:4001"
+# Yield execution to venv command
+$*

setup.cfg

@@ -25,6 +25,7 @@ packages =
 
 [entry_points]
 tooz.backends =
+    etcd = tooz.drivers.etcd:EtcdDriver
     kazoo = tooz.drivers.zookeeper:KazooDriver
     zake = tooz.drivers.zake:ZakeDriver
     memcached = tooz.drivers.memcached:MemcachedDriver

tools/compat-matrix.py

@@ -33,6 +33,7 @@ def print_methods(methods):
 
 driver_tpl = ":py:class:`~tooz.drivers.%s`"
 driver_class_names = [
+    "etcd.EtcdDriver",
     "file.FileDriver",
     "ipc.IPCDriver",
     "memcached.MemcachedDriver",
@@ -71,6 +72,7 @@ print_header("Driver support", delim="-")
 print("")
 grouping_table = [
     [
+        "No",  # Etcd
         "Yes",  # File
         "No",  # IPC
         "Yes",  # Memcached
@@ -101,6 +103,7 @@ print_header("Driver support", delim="-")
 print("")
 leader_table = [
     [
+        "No",  # Etcd
         "No",  # File
         "No",  # IPC
         "Yes",  # Memcached
@@ -128,6 +131,7 @@ print_header("Driver support", delim="-")
 print("")
 lock_table = [
     [
+        "Yes",  # Etcd
         "Yes",  # File
         "Yes",  # IPC
         "Yes",  # Memcached

tooz/drivers/etcd.py

@@ -0,0 +1,201 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+
+from oslo_utils import timeutils
+import requests
+import six
+
+import tooz
+from tooz import coordination
+from tooz import locking
+from tooz import utils
+
+
+LOG = logging.getLogger(__name__)
+
+
+def _translate_failures(func):
+    """Translates common requests exceptions into tooz exceptions."""
+
+    @six.wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except requests.exceptions.RequestException as e:
+            coordination.raise_with_cause(coordination.ToozConnectionError,
+                                          utils.exception_message(e),
+                                          cause=e)
+
+    return wrapper
+
+
+class _Client(object):
+    def __init__(self, host, port, protocol):
+        self.host = host
+        self.port = port
+        self.protocol = protocol
+        self.session = requests.Session()
+
+    @property
+    def base_url(self):
+        return self.protocol + '://' + self.host + ':' + str(self.port)
+
+    def get_url(self, path):
+        return self.base_url + '/v2/' + path.lstrip("/")
+
+    def get(self, url, **kwargs):
+        return self.session.get(self.get_url(url), **kwargs).json()
+
+    def put(self, url, **kwargs):
+        return self.session.put(self.get_url(url), **kwargs).json()
+
+    def delete(self, url, **kwargs):
+        return self.session.delete(self.get_url(url), **kwargs).json()
+
+    def self_stats(self):
+        return self.session.get(self.get_url("/stats/self"))
+
+
+class EtcdLock(locking.Lock):
+    _TOOZ_LOCK_PREFIX = "tooz_locks"
+
+    def __init__(self, name, coord, client, ttl=60):
+        super(EtcdLock, self).__init__(name)
+        self.client = client
+        self.coord = coord
+        self.lock = None
+        self.ttl = ttl
+        # NOTE(jd) sha1 of the name to be sure it works with any string?
+        self._lock_url = (
+            "/keys/" + self._TOOZ_LOCK_PREFIX + "/" + name.decode('ascii')
+        )
+
+    def acquire(self, blocking=True):
+        if isinstance(blocking, bool):
+            watch = None
+        else:
+            watch = timeutils.StopWatch(duration=blocking)
+            watch.start()
+
+        while True:
+            try:
+                reply = self.client.put(
+                    self._lock_url,
+                    timeout=watch.leftover() if watch else None,
+                    data={"ttl": self.ttl,
+                          "prevExist": "false"})
+            except requests.exceptions.RequestException:
+                if watch and watch.leftover() == 0:
+                    return False
+
+            # We got the lock!
+            if reply.get("errorCode") is None:
+                self.coord._acquired_locks.append(self)
+                return True
+
+            # We didn't get the lock and we don't want to wait
+            if blocking is False:
+                return False
+
+            # Ok, so let's wait a bit (or forever!)
+            try:
+                reply = self.client.get(
+                    self._lock_url
+                    + "?wait=true&waitIndex=%d" % reply['index'],
+                    timeout=watch.leftover() if watch else None)
+            except requests.exceptions.RequestException:
+                if watch and watch.expired():
+                    return False
+
+    @_translate_failures
+    def release(self):
+        if self in self.coord._acquired_locks:
+            reply = self.client.delete(self._lock_url)
+            if reply.get("errorCode") is None:
+                self.coord._acquired_locks.remove(self)
+                return True
+        return False
+
+    @_translate_failures
+    def heartbeat(self):
+        """Keep the lock alive."""
+        poked = self.client.put(self._lock_url,
+                                data={"ttl": self.ttl,
+                                      "prevExist": "true"})
+        errorcode = poked.get("errorCode")
+        if errorcode:
+            LOG.warn("Unable to heartbeat by updating key '%s' with extended"
+                     " expiry of %s seconds: %d, %s", self.name, self.ttl,
+                     errorcode, poked.get("message"))
+
+
+class EtcdDriver(coordination.CoordinationDriver):
+    """An etcd based driver.
+
+    This driver uses etcd provide the coordination driver semantics and
+    required API(s).
+    """
+
+    #: Default socket/lock/member/leader timeout used when none is provided.
+    DEFAULT_TIMEOUT = 30
+
+    def __init__(self, member_id, parsed_url, options):
+        super(EtcdDriver, self).__init__()
+        options = utils.collapse(options)
+        self.client = _Client(host=parsed_url.hostname,
+                              port=parsed_url.port,
+                              protocol=options.get('protocol', 'http'))
+        default_timeout = options.get('timeout', self.DEFAULT_TIMEOUT)
+        self.lock_timeout = int(options.get(
+            'lock_timeout', default_timeout))
+        self._acquired_locks = []
+
+    def _start(self):
+        try:
+            self.client.self_stats()
+        except requests.exceptions.ConnectionError as e:
+            raise coordination.ToozConnectionError(utils.exception_message(e))
+
+    def get_lock(self, name):
+        return EtcdLock(name, self, self.client, self.lock_timeout)
+
+    def heartbeat(self):
+        for lock in self._acquired_locks:
+            lock.heartbeat()
+
+    @staticmethod
+    def watch_join_group(group_id, callback):
+        raise tooz.NotImplemented
+
+    @staticmethod
+    def unwatch_join_group(group_id, callback):
+        raise tooz.NotImplemented
+
+    @staticmethod
+    def watch_leave_group(group_id, callback):
+        raise tooz.NotImplemented
+
+    @staticmethod
+    def unwatch_leave_group(group_id, callback):
+        raise tooz.NotImplemented
+
+    @staticmethod
+    def watch_elected_as_leader(group_id, callback):
+        raise tooz.NotImplemented
+
+    @staticmethod
+    def unwatch_elected_as_leader(group_id, callback):
+        raise tooz.NotImplemented

tooz/tests/test_coordination.py

@@ -60,6 +60,8 @@ class TestAPI(testscenarios.TestWithScenarios,
                    'bad_url': 'mysql://localhost:1'}),
         ('zookeeper', {'url': os.getenv("TOOZ_TEST_ZOOKEEPER_URL"),
                        'bad_url': 'zookeeper://localhost:1'}),
+        ('etcd', {'url': os.getenv("TOOZ_TEST_ETCD_URL"),
+                  'bad_url': 'etcd://localhost:1'})
     ]
 
     def assertRaisesAny(self, exc_classes, callable_obj, *args, **kwargs):

tox.ini

@@ -56,6 +56,13 @@ commands = {toxinidir}/setup-mysql-env.sh python setup.py testr --slowest --test
 basepython = python3.4
 commands = {toxinidir}/setup-mysql-env.sh python setup.py testr --slowest --testr-args="{posargs}"
 
+[testenv:py27-etcd]
+commands = {toxinidir}/setup-etcd-env.sh python setup.py testr --slowest --testr-args="{posargs}"
+
+[testenv:py34-etcd]
+basepython = python3.4
+commands = {toxinidir}/setup-etcd-env.sh python setup.py testr --slowest --testr-args="{posargs}"
+
 [testenv:cover]
 commands = python setup.py testr --slowest --coverage --testr-args="{posargs}"