Update sentinel support to allow multiple sentinel hosts

Multiple sentinel hosts are now allowed by listing the additional
hosts as multiple 'sentinel_fallback' parameters. These are combined
with the first sentinel host to create a list that is provided to
Sentinel class constructor.

This provides safety in the case when the first (or any other) sentinel
host goes down.

Tests are run twice: once with fallbacks, once without.

Change-Id: Ie0a134f02617e8552a87c8bcd49996daaba07f4e
This commit is contained in:
Chris Dent 2015-01-12 13:32:35 +00:00
parent a2216e383e
commit 886aa622b0
3 changed files with 50 additions and 12 deletions

View File

@ -15,7 +15,7 @@ API, some of them have different properties:
on timeout (heartbeats, locks, etc) so are less resilient than other on timeout (heartbeats, locks, etc) so are less resilient than other
backends. backends.
* `redis`_ is a basic implementation and provides some resiliency * `redis`_ is a basic implementation and provides reasonable resiliency
when used with redis-sentinel. A lot of the features provided in tooz are when used with redis-sentinel. A lot of the features provided in tooz are
based on timeout (heartbeats, locks, etc) so are less resilient than other based on timeout (heartbeats, locks, etc) so are less resilient than other
backends. backends.

View File

@ -1,7 +1,8 @@
#!/bin/bash #!/bin/bash
set -x -e set -x -e
CONFFILE=$(mktemp -t tooz-sentinel-XXXXXX) SENTINEL_PORTS="26381 26382 26383"
CONFFILES=()
function clean_exit(){ function clean_exit(){
local error_code="$?" local error_code="$?"
@ -10,29 +11,45 @@ function clean_exit(){
kill $(jobs -p) kill $(jobs -p)
fi fi
wait $spawned wait $spawned
rm $CONFFILE || true rm /tmp/sentinel.2638[123].log || true
rm /tmp/sentinel.26381.log || true rm ${CONFFILES[@]} || true
return $error_code return $error_code
} }
cat > $CONFFILE <<EOF function write_conf_file() {
port 26381 local port=$1
local conffile=$(mktemp -t tooz-sentinel-$port-XXXXXX)
cat > $conffile <<EOF
port $port
dir /tmp dir /tmp
sentinel monitor mainbarn 127.0.0.1 6381 1 sentinel monitor mainbarn 127.0.0.1 6381 2
sentinel down-after-milliseconds mainbarn 80000 sentinel down-after-milliseconds mainbarn 80000
sentinel parallel-syncs mainbarn 1 sentinel parallel-syncs mainbarn 1
sentinel failover-timeout mainbarn 180000 sentinel failover-timeout mainbarn 180000
logfile /tmp/sentinel.26381.log logfile /tmp/sentinel.$port.log
EOF EOF
echo $conffile
}
trap "clean_exit" EXIT trap "clean_exit" EXIT
# If we can't find either redis-server or redis-sentinel, exit. # If we can't find either redis-server or redis-sentinel, exit.
redis_bin=$(which redis-server) redis_bin=$(which redis-server)
sentinel_bin=$(which redis-sentinel) sentinel_bin=$(which redis-sentinel)
$redis_bin --port 6381 &
$sentinel_bin $CONFFILE &
# start redis
$redis_bin --port 6381 &
# start the sentinels
for port in $SENTINEL_PORTS; do
conffile=$(write_conf_file $port)
$sentinel_bin $conffile &
CONFFILES+=($conffile)
done
# Test a first time without sentinel fallbacks
export TOOZ_TEST_REDIS_URL="redis://localhost:26381?sentinel=mainbarn&timeout=5" export TOOZ_TEST_REDIS_URL="redis://localhost:26381?sentinel=mainbarn&timeout=5"
# Yield execution to venv command $*
# Test a second time with sentinel fallbacks
export TOOZ_TEST_REDIS_URL="redis://localhost:26381?sentinel=mainbarn&sentinel_fallback=localhost:26382&sentinel_fallback=localhost:26383&timeout=5"
$* $*

View File

@ -151,6 +151,14 @@ class RedisDriver(coordination.CoordinationDriver):
redis://<sentinel host>:<sentinel port>?sentinel=<master name> redis://<sentinel host>:<sentinel port>?sentinel=<master name>
Additional sentinel hosts are listed with mutiple ``sentinel_fallback``
parameters as follows:
redis://<sentinel host>:<sentinel port>?sentinel=<master name>&
sentinel_fallback=<other sentinel host>:<sentinel port>&
sentinel_fallback=<other sentinel host>:<sentinel port>&
sentinel_fallback=<other sentinel host>:<sentinel port>
Further resources/links: Further resources/links:
- http://redis.io/ - http://redis.io/
@ -202,6 +210,10 @@ class RedisDriver(coordination.CoordinationDriver):
'ssl_certfile', 'ssl_certfile',
'ssl_keyfile', 'ssl_keyfile',
'sentinel', 'sentinel',
'sentinel_fallback',
])
_CLIENT_LIST_ARGS = frozenset([
'sentinel_fallback',
]) ])
_CLIENT_BOOL_ARGS = frozenset([ _CLIENT_BOOL_ARGS = frozenset([
'retry_on_timeout', 'retry_on_timeout',
@ -317,6 +329,8 @@ class RedisDriver(coordination.CoordinationDriver):
# redis://localhost:6379?timeout=5&timeout=2 # redis://localhost:6379?timeout=5&timeout=2
if a in cls._CLIENT_BOOL_ARGS: if a in cls._CLIENT_BOOL_ARGS:
v = strutils.bool_from_string(options[a][-1]) v = strutils.bool_from_string(options[a][-1])
elif a in cls._CLIENT_LIST_ARGS:
v = options[a]
elif a in cls._CLIENT_INT_ARGS: elif a in cls._CLIENT_INT_ARGS:
v = int(options[a][-1]) v = int(options[a][-1])
else: else:
@ -328,14 +342,21 @@ class RedisDriver(coordination.CoordinationDriver):
# Ask the sentinel for the current master if there is a # Ask the sentinel for the current master if there is a
# sentinel arg. # sentinel arg.
if 'sentinel' in kwargs: if 'sentinel' in kwargs:
sentinel_hosts = [
tuple(fallback.split(':'))
for fallback in kwargs.get('sentinel_fallback', [])
]
sentinel_hosts.insert(0, (kwargs['host'], kwargs['port']))
sentinel_server = sentinel.Sentinel( sentinel_server = sentinel.Sentinel(
[(kwargs['host'], kwargs['port'])], sentinel_hosts,
socket_timeout=kwargs['socket_timeout']) socket_timeout=kwargs['socket_timeout'])
master_host, master_port = sentinel_server.discover_master( master_host, master_port = sentinel_server.discover_master(
kwargs['sentinel']) kwargs['sentinel'])
kwargs['host'] = master_host kwargs['host'] = master_host
kwargs['port'] = master_port kwargs['port'] = master_port
del kwargs['sentinel'] del kwargs['sentinel']
if 'sentinel_fallback' in kwargs:
del kwargs['sentinel_fallback']
return redis.StrictRedis(**kwargs) return redis.StrictRedis(**kwargs)
def _start(self): def _start(self):