Update sentinel support to allow multiple sentinel hosts
Multiple sentinel hosts are now allowed by listing the additional hosts as multiple 'sentinel_fallback' parameters. These are combined with the first sentinel host to create a list that is provided to Sentinel class constructor. This provides safety in the case when the first (or any other) sentinel host goes down. Tests are run twice: once with fallbacks, once without. Change-Id: Ie0a134f02617e8552a87c8bcd49996daaba07f4e
This commit is contained in:
parent
a2216e383e
commit
886aa622b0
|
@ -15,7 +15,7 @@ API, some of them have different properties:
|
||||||
on timeout (heartbeats, locks, etc) so are less resilient than other
|
on timeout (heartbeats, locks, etc) so are less resilient than other
|
||||||
backends.
|
backends.
|
||||||
|
|
||||||
* `redis`_ is a basic implementation and provides some resiliency
|
* `redis`_ is a basic implementation and provides reasonable resiliency
|
||||||
when used with redis-sentinel. A lot of the features provided in tooz are
|
when used with redis-sentinel. A lot of the features provided in tooz are
|
||||||
based on timeout (heartbeats, locks, etc) so are less resilient than other
|
based on timeout (heartbeats, locks, etc) so are less resilient than other
|
||||||
backends.
|
backends.
|
||||||
|
|
|
@ -1,7 +1,8 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
set -x -e
|
set -x -e
|
||||||
|
|
||||||
CONFFILE=$(mktemp -t tooz-sentinel-XXXXXX)
|
SENTINEL_PORTS="26381 26382 26383"
|
||||||
|
CONFFILES=()
|
||||||
|
|
||||||
function clean_exit(){
|
function clean_exit(){
|
||||||
local error_code="$?"
|
local error_code="$?"
|
||||||
|
@ -10,29 +11,45 @@ function clean_exit(){
|
||||||
kill $(jobs -p)
|
kill $(jobs -p)
|
||||||
fi
|
fi
|
||||||
wait $spawned
|
wait $spawned
|
||||||
rm $CONFFILE || true
|
rm /tmp/sentinel.2638[123].log || true
|
||||||
rm /tmp/sentinel.26381.log || true
|
rm ${CONFFILES[@]} || true
|
||||||
return $error_code
|
return $error_code
|
||||||
}
|
}
|
||||||
|
|
||||||
cat > $CONFFILE <<EOF
|
function write_conf_file() {
|
||||||
port 26381
|
local port=$1
|
||||||
|
local conffile=$(mktemp -t tooz-sentinel-$port-XXXXXX)
|
||||||
|
cat > $conffile <<EOF
|
||||||
|
port $port
|
||||||
dir /tmp
|
dir /tmp
|
||||||
sentinel monitor mainbarn 127.0.0.1 6381 1
|
sentinel monitor mainbarn 127.0.0.1 6381 2
|
||||||
sentinel down-after-milliseconds mainbarn 80000
|
sentinel down-after-milliseconds mainbarn 80000
|
||||||
sentinel parallel-syncs mainbarn 1
|
sentinel parallel-syncs mainbarn 1
|
||||||
sentinel failover-timeout mainbarn 180000
|
sentinel failover-timeout mainbarn 180000
|
||||||
logfile /tmp/sentinel.26381.log
|
logfile /tmp/sentinel.$port.log
|
||||||
EOF
|
EOF
|
||||||
|
echo $conffile
|
||||||
|
}
|
||||||
|
|
||||||
trap "clean_exit" EXIT
|
trap "clean_exit" EXIT
|
||||||
|
|
||||||
# If we can't find either redis-server or redis-sentinel, exit.
|
# If we can't find either redis-server or redis-sentinel, exit.
|
||||||
redis_bin=$(which redis-server)
|
redis_bin=$(which redis-server)
|
||||||
sentinel_bin=$(which redis-sentinel)
|
sentinel_bin=$(which redis-sentinel)
|
||||||
$redis_bin --port 6381 &
|
|
||||||
$sentinel_bin $CONFFILE &
|
|
||||||
|
|
||||||
|
# start redis
|
||||||
|
$redis_bin --port 6381 &
|
||||||
|
|
||||||
|
# start the sentinels
|
||||||
|
for port in $SENTINEL_PORTS; do
|
||||||
|
conffile=$(write_conf_file $port)
|
||||||
|
$sentinel_bin $conffile &
|
||||||
|
CONFFILES+=($conffile)
|
||||||
|
done
|
||||||
|
|
||||||
|
# Test a first time without sentinel fallbacks
|
||||||
export TOOZ_TEST_REDIS_URL="redis://localhost:26381?sentinel=mainbarn&timeout=5"
|
export TOOZ_TEST_REDIS_URL="redis://localhost:26381?sentinel=mainbarn&timeout=5"
|
||||||
# Yield execution to venv command
|
$*
|
||||||
|
# Test a second time with sentinel fallbacks
|
||||||
|
export TOOZ_TEST_REDIS_URL="redis://localhost:26381?sentinel=mainbarn&sentinel_fallback=localhost:26382&sentinel_fallback=localhost:26383&timeout=5"
|
||||||
$*
|
$*
|
||||||
|
|
|
@ -151,6 +151,14 @@ class RedisDriver(coordination.CoordinationDriver):
|
||||||
|
|
||||||
redis://<sentinel host>:<sentinel port>?sentinel=<master name>
|
redis://<sentinel host>:<sentinel port>?sentinel=<master name>
|
||||||
|
|
||||||
|
Additional sentinel hosts are listed with mutiple ``sentinel_fallback``
|
||||||
|
parameters as follows:
|
||||||
|
|
||||||
|
redis://<sentinel host>:<sentinel port>?sentinel=<master name>&
|
||||||
|
sentinel_fallback=<other sentinel host>:<sentinel port>&
|
||||||
|
sentinel_fallback=<other sentinel host>:<sentinel port>&
|
||||||
|
sentinel_fallback=<other sentinel host>:<sentinel port>
|
||||||
|
|
||||||
Further resources/links:
|
Further resources/links:
|
||||||
|
|
||||||
- http://redis.io/
|
- http://redis.io/
|
||||||
|
@ -202,6 +210,10 @@ class RedisDriver(coordination.CoordinationDriver):
|
||||||
'ssl_certfile',
|
'ssl_certfile',
|
||||||
'ssl_keyfile',
|
'ssl_keyfile',
|
||||||
'sentinel',
|
'sentinel',
|
||||||
|
'sentinel_fallback',
|
||||||
|
])
|
||||||
|
_CLIENT_LIST_ARGS = frozenset([
|
||||||
|
'sentinel_fallback',
|
||||||
])
|
])
|
||||||
_CLIENT_BOOL_ARGS = frozenset([
|
_CLIENT_BOOL_ARGS = frozenset([
|
||||||
'retry_on_timeout',
|
'retry_on_timeout',
|
||||||
|
@ -317,6 +329,8 @@ class RedisDriver(coordination.CoordinationDriver):
|
||||||
# redis://localhost:6379?timeout=5&timeout=2
|
# redis://localhost:6379?timeout=5&timeout=2
|
||||||
if a in cls._CLIENT_BOOL_ARGS:
|
if a in cls._CLIENT_BOOL_ARGS:
|
||||||
v = strutils.bool_from_string(options[a][-1])
|
v = strutils.bool_from_string(options[a][-1])
|
||||||
|
elif a in cls._CLIENT_LIST_ARGS:
|
||||||
|
v = options[a]
|
||||||
elif a in cls._CLIENT_INT_ARGS:
|
elif a in cls._CLIENT_INT_ARGS:
|
||||||
v = int(options[a][-1])
|
v = int(options[a][-1])
|
||||||
else:
|
else:
|
||||||
|
@ -328,14 +342,21 @@ class RedisDriver(coordination.CoordinationDriver):
|
||||||
# Ask the sentinel for the current master if there is a
|
# Ask the sentinel for the current master if there is a
|
||||||
# sentinel arg.
|
# sentinel arg.
|
||||||
if 'sentinel' in kwargs:
|
if 'sentinel' in kwargs:
|
||||||
|
sentinel_hosts = [
|
||||||
|
tuple(fallback.split(':'))
|
||||||
|
for fallback in kwargs.get('sentinel_fallback', [])
|
||||||
|
]
|
||||||
|
sentinel_hosts.insert(0, (kwargs['host'], kwargs['port']))
|
||||||
sentinel_server = sentinel.Sentinel(
|
sentinel_server = sentinel.Sentinel(
|
||||||
[(kwargs['host'], kwargs['port'])],
|
sentinel_hosts,
|
||||||
socket_timeout=kwargs['socket_timeout'])
|
socket_timeout=kwargs['socket_timeout'])
|
||||||
master_host, master_port = sentinel_server.discover_master(
|
master_host, master_port = sentinel_server.discover_master(
|
||||||
kwargs['sentinel'])
|
kwargs['sentinel'])
|
||||||
kwargs['host'] = master_host
|
kwargs['host'] = master_host
|
||||||
kwargs['port'] = master_port
|
kwargs['port'] = master_port
|
||||||
del kwargs['sentinel']
|
del kwargs['sentinel']
|
||||||
|
if 'sentinel_fallback' in kwargs:
|
||||||
|
del kwargs['sentinel_fallback']
|
||||||
return redis.StrictRedis(**kwargs)
|
return redis.StrictRedis(**kwargs)
|
||||||
|
|
||||||
def _start(self):
|
def _start(self):
|
||||||
|
|
Loading…
Reference in New Issue