Update sentinel support to allow multiple sentinel hosts

Multiple sentinel hosts are now allowed by listing the additional
hosts as multiple 'sentinel_fallback' parameters. These are combined
with the first sentinel host to create a list that is provided to
Sentinel class constructor.

This provides safety in the case when the first (or any other) sentinel
host goes down.

Tests are run twice: once with fallbacks, once without.

Change-Id: Ie0a134f02617e8552a87c8bcd49996daaba07f4e
This commit is contained in:
Chris Dent 2015-01-12 13:32:35 +00:00
parent a2216e383e
commit 886aa622b0
3 changed files with 50 additions and 12 deletions

View File

@ -15,7 +15,7 @@ API, some of them have different properties:
on timeout (heartbeats, locks, etc) so are less resilient than other
backends.
* `redis`_ is a basic implementation and provides some resiliency
* `redis`_ is a basic implementation and provides reasonable resiliency
when used with redis-sentinel. A lot of the features provided in tooz are
based on timeout (heartbeats, locks, etc) so are less resilient than other
backends.

View File

@ -1,7 +1,8 @@
#!/bin/bash
set -x -e
CONFFILE=$(mktemp -t tooz-sentinel-XXXXXX)
SENTINEL_PORTS="26381 26382 26383"
CONFFILES=()
function clean_exit(){
local error_code="$?"
@ -10,29 +11,45 @@ function clean_exit(){
kill $(jobs -p)
fi
wait $spawned
rm $CONFFILE || true
rm /tmp/sentinel.26381.log || true
rm /tmp/sentinel.2638[123].log || true
rm ${CONFFILES[@]} || true
return $error_code
}
cat > $CONFFILE <<EOF
port 26381
function write_conf_file() {
local port=$1
local conffile=$(mktemp -t tooz-sentinel-$port-XXXXXX)
cat > $conffile <<EOF
port $port
dir /tmp
sentinel monitor mainbarn 127.0.0.1 6381 1
sentinel monitor mainbarn 127.0.0.1 6381 2
sentinel down-after-milliseconds mainbarn 80000
sentinel parallel-syncs mainbarn 1
sentinel failover-timeout mainbarn 180000
logfile /tmp/sentinel.26381.log
logfile /tmp/sentinel.$port.log
EOF
echo $conffile
}
trap "clean_exit" EXIT
# If we can't find either redis-server or redis-sentinel, exit.
redis_bin=$(which redis-server)
sentinel_bin=$(which redis-sentinel)
$redis_bin --port 6381 &
$sentinel_bin $CONFFILE &
# start redis
$redis_bin --port 6381 &
# start the sentinels
for port in $SENTINEL_PORTS; do
conffile=$(write_conf_file $port)
$sentinel_bin $conffile &
CONFFILES+=($conffile)
done
# Test a first time without sentinel fallbacks
export TOOZ_TEST_REDIS_URL="redis://localhost:26381?sentinel=mainbarn&timeout=5"
# Yield execution to venv command
$*
# Test a second time with sentinel fallbacks
export TOOZ_TEST_REDIS_URL="redis://localhost:26381?sentinel=mainbarn&sentinel_fallback=localhost:26382&sentinel_fallback=localhost:26383&timeout=5"
$*

View File

@ -151,6 +151,14 @@ class RedisDriver(coordination.CoordinationDriver):
redis://<sentinel host>:<sentinel port>?sentinel=<master name>
Additional sentinel hosts are listed with mutiple ``sentinel_fallback``
parameters as follows:
redis://<sentinel host>:<sentinel port>?sentinel=<master name>&
sentinel_fallback=<other sentinel host>:<sentinel port>&
sentinel_fallback=<other sentinel host>:<sentinel port>&
sentinel_fallback=<other sentinel host>:<sentinel port>
Further resources/links:
- http://redis.io/
@ -202,6 +210,10 @@ class RedisDriver(coordination.CoordinationDriver):
'ssl_certfile',
'ssl_keyfile',
'sentinel',
'sentinel_fallback',
])
_CLIENT_LIST_ARGS = frozenset([
'sentinel_fallback',
])
_CLIENT_BOOL_ARGS = frozenset([
'retry_on_timeout',
@ -317,6 +329,8 @@ class RedisDriver(coordination.CoordinationDriver):
# redis://localhost:6379?timeout=5&timeout=2
if a in cls._CLIENT_BOOL_ARGS:
v = strutils.bool_from_string(options[a][-1])
elif a in cls._CLIENT_LIST_ARGS:
v = options[a]
elif a in cls._CLIENT_INT_ARGS:
v = int(options[a][-1])
else:
@ -328,14 +342,21 @@ class RedisDriver(coordination.CoordinationDriver):
# Ask the sentinel for the current master if there is a
# sentinel arg.
if 'sentinel' in kwargs:
sentinel_hosts = [
tuple(fallback.split(':'))
for fallback in kwargs.get('sentinel_fallback', [])
]
sentinel_hosts.insert(0, (kwargs['host'], kwargs['port']))
sentinel_server = sentinel.Sentinel(
[(kwargs['host'], kwargs['port'])],
sentinel_hosts,
socket_timeout=kwargs['socket_timeout'])
master_host, master_port = sentinel_server.discover_master(
kwargs['sentinel'])
kwargs['host'] = master_host
kwargs['port'] = master_port
del kwargs['sentinel']
if 'sentinel_fallback' in kwargs:
del kwargs['sentinel_fallback']
return redis.StrictRedis(**kwargs)
def _start(self):