Use pifpaf to setup daemons

Depends-On: Id3e6b694bb186724517599cd9875ad80ceeee053
Change-Id: I2265b4b83f136f2eeb21e86cda1c7ec36401d0a8
This commit is contained in:
Julien Danjou 2016-02-12 15:47:51 +01:00
parent 6daa485eeb
commit 4906e72416
12 changed files with 55 additions and 384 deletions

View File

@ -1,108 +1,27 @@
#!/bin/bash
set -eux
set -x
set -e
CONSUL_DOWN_DIR=`mktemp -d`
CONSUL_BIN_DIR=`mktemp -d`
CONSUL_TMP_DATA_DIR=`mktemp -d`
CONSUL_VERSION="0.6.3"
CONSUL_RELEASE_URL="https://releases.hashicorp.com/consul"
if [ ! -d "$CONSUL_DOWN_DIR" ]; then
mkdir -p $CONSUL_DOWN_DIR
if [ -z "$(which consul)" ]; then
CONSUL_VERSION=0.6.3
CONSUL_RELEASE_URL=https://releases.hashicorp.com/consul
case `uname -s` in
Darwin)
consul_file="consul_${CONSUL_VERSION}_darwin_amd64.zip"
;;
Linux)
consul_file="consul_${CONSUL_VERSION}_linux_amd64.zip"
;;
*)
echo "Unknown operating system"
exit 1
;;
esac
consul_dir=`basename $consul_file .zip`
mkdir -p $consul_dir
curl -L $CONSUL_RELEASE_URL/$CONSUL_VERSION/$consul_file > $consul_dir/$consul_file
unzip $consul_dir/$consul_file -d $consul_dir
export PATH=$PATH:$consul_dir
fi
if [ ! -d "$CONSUL_BIN_DIR" ]; then
mkdir -p $CONSUL_BIN_DIR
fi
if [ ! -d "$CONSUL_TMP_DATA_DIR" ]; then
mkdir -p $CONSUL_TMP_DATA_DIR
fi
function clean_exit(){
local error_code="$?"
local spawned=$(jobs -p)
if [ -n "$spawned" ]; then
kill $spawned
fi
rm -rf $CONSUL_TMP_DATA_DIR
rm -rf $CONSUL_BIN_DIR
rm -rf $CONSUL_DOWN_DIR
return $error_code
}
function get_leader(){
local leader=""
leader=$(curl -s http://127.0.0.1:8500/v1/status/leader)
if [ $? -ne 0 ]; then
return 1
else
echo $leader | python -c "import sys;\
import json;\
print(json.loads(sys.stdin.read()))"
return 0
fi
}
function wait_until_up(){
local leader=`get_leader`
while [ -z "$leader" ]; do
echo "Waiting for consul to respond to a leader http request"
sleep 1
leader=`get_leader`
done
}
function download_and_expand_consul() {
if [ "$(uname)" == "Darwin" ]; then
local consul_file="consul_${CONSUL_VERSION}_darwin_amd64.zip"
elif [ "$(uname -a | cut -d" " -f1)" == "Linux" ]; then
local consul_file="consul_${CONSUL_VERSION}_linux_amd64.zip"
else
echo "Unknown operating system '$(uname -a)'"
exit 1
fi
wget $CONSUL_RELEASE_URL/$CONSUL_VERSION/$consul_file \
--directory-prefix $CONSUL_DOWN_DIR
if [ $? -ne 0 ]; then
echo "Unable to download consul"
exit 1
fi
unzip $CONSUL_DOWN_DIR/$consul_file -d $CONSUL_BIN_DIR
}
function get_consul_version() {
local consul_bin="$1"
local consul_version=`$consul_bin --version | \
head -n1 | cut -d" " -f2 | \
cut -d"v" -f2`
echo $consul_version
}
trap "clean_exit" EXIT
CONSUL_BIN=`which consul || true`
if [ -z "$CONSUL_BIN" ]; then
echo "Downloading consul $CONSUL_VERSION"
download_and_expand_consul
CONSUL_BIN=$CONSUL_BIN_DIR/consul
if [ ! -e "$CONSUL_BIN" ]; then
echo "Consul executable does not exist (even after downloading it)"
exit 1
fi
else
CONSUL_VERSION=`get_consul_version "$CONSUL_BIN"`
echo "Consul $CONSUL_VERSION is already installed"
fi
$CONSUL_BIN agent -server -bootstrap-expect 1 -data-dir $CONSUL_TMP_DATA_DIR -node=agent-one -bind=127.0.0.1 &
# Give some time for the agent to elect a leader, and startup...
# TODO(harlowja): there has to be a better way to do this, there doesn't
# seem
wait_until_up
export TOOZ_TEST_CONSUL_URL="consul://localhost:8500/v1"
# Yield execution to venv command
$*

View File

@ -1,16 +1,6 @@
#!/bin/bash
set -eux
clean_exit() {
local error_code="$?"
kill $(jobs -p)
return $error_code
}
trap clean_exit EXIT
if [ -n "$(which etcd)" ]; then
etcd &
else
if [ -z "$(which etcd)" ]; then
ETCD_VERSION=2.2.2
case `uname -s` in
Darwin)
@ -35,9 +25,7 @@ else
esac
TARBALL_NAME=etcd-v${ETCD_VERSION}-$OS-$MACHINE
test ! -d "$TARBALL_NAME" && curl -L https://github.com/coreos/etcd/releases/download/v${ETCD_VERSION}/${TARBALL_NAME}.${SUFFIX} | tar xz
$TARBALL_NAME/etcd &
export PATH=$PATH:$TARBALL_NAME
fi
export TOOZ_TEST_ETCD_URL="etcd://localhost:4001"
# Yield execution to venv command
$*

View File

@ -1,22 +0,0 @@
#!/bin/bash
set -x -e
function clean_exit(){
local error_code="$?"
local spawned=$(jobs -p)
if [ -n "$spawned" ]; then
kill $(jobs -p)
fi
return $error_code
}
trap "clean_exit" EXIT
memcached_bin=$(which memcached || true)
if [ -n "$memcached_bin" ]; then
$memcached_bin -p 11212 &
fi
export TOOZ_TEST_MEMCACHED_URL="memcached://localhost:11212?timeout=5"
# Yield execution to venv command
$*

View File

@ -1,49 +0,0 @@
#!/bin/bash
set -x -e
clean_exit () {
local error_code="$?"
kill $(jobs -p)
rm -rf ${MYSQL_DATA}
return $error_code
}
wait_for_line () {
while read line
do
echo "$line" | grep -q "$1" && break
done < "$2"
# Read the fifo for ever otherwise process would block
cat "$2" >/dev/null &
}
get_random_port () {
PORT=13306
while netstat -atwn | grep "^.*:${PORT}.*:\*\s*LISTEN\s*$"
do
PORT=$(( ${PORT} + 1 ))
done
}
trap "clean_exit" EXIT
# On systems like Fedora here's where mysqld can be found
export PATH=$PATH:/usr/libexec
# Start MySQL process for tests
MYSQL_DATA=`mktemp -d -t tooz-mysql-XXXXX`
mkfifo ${MYSQL_DATA}/out
# Initialize MySQL Data Directory
mysql_install_db --user=${USER} --ldata=${MYSQL_DATA}
# Get random unused port for mysql
get_random_port
# Start mysqld with networking (i.e. - allow connection with TCP/IP)
mysqld --no-defaults --datadir=${MYSQL_DATA} --port=${PORT} --pid-file=${MYSQL_DATA}/mysql.pid --socket=${MYSQL_DATA}/mysql.socket --slow-query-log-file=${MYSQL_DATA}/mysql-slow.log --skip-grant-tables &> ${MYSQL_DATA}/out &
# Wait for MySQL to start listening to connections
wait_for_line "mysqld: ready for connections." ${MYSQL_DATA}/out
# The default root password is blank
mysql --host=localhost -S ${MYSQL_DATA}/mysql.socket -e 'CREATE DATABASE IF NOT EXISTS test;'
export TOOZ_TEST_MYSQL_URL="mysql://root@localhost:${PORT}/test"
# Yield execution to venv command
$*

View File

@ -1,23 +0,0 @@
#!/bin/bash
set -x -e
function clean_exit() {
local error_code="$?"
${PGSQL_PATH}/pg_ctl -w -D ${PGSQL_DATA} -o "-p $PGSQL_PORT" stop
rm -rf ${PGSQL_DATA}
return $error_code
}
trap "clean_exit" EXIT
# Start PostgreSQL process for tests
PGSQL_DATA=`mktemp -d -t tooz-pgsql-XXXXX`
PGSQL_PATH=`pg_config --bindir`
PGSQL_PORT=9825
${PGSQL_PATH}/initdb ${PGSQL_DATA}
${PGSQL_PATH}/pg_ctl -w -D ${PGSQL_DATA} -o "-k ${PGSQL_DATA} -p ${PGSQL_PORT}" start
# Wait for PostgreSQL to start listening to connections
export TOOZ_TEST_PGSQL_URL="postgresql:///?host=${PGSQL_DATA}&port=${PGSQL_PORT}&dbname=template1"
# Yield execution to venv command
$*

View File

@ -1,22 +0,0 @@
#!/bin/bash
set -x -e
function clean_exit(){
local error_code="$?"
local spawned=$(jobs -p)
if [ -n "$spawned" ]; then
kill $(jobs -p)
fi
return $error_code
}
trap "clean_exit" EXIT
redis_bin=$(which redis-server || true)
if [ -n "$redis_bin" ]; then
$redis_bin --port 6380 &
fi
export TOOZ_TEST_REDIS_URL="redis://localhost:6380?timeout=5"
# Yield execution to venv command
$*

View File

@ -1,56 +0,0 @@
#!/bin/bash
set -x -e
SENTINEL_PORTS="26381 26382 26383"
CONFFILES=()
TMPDIR=${TMPDIR:/tmp}
function clean_exit(){
local error_code="$?"
local spawned=$(jobs -p)
if [ -n "$spawned" ]; then
kill $(jobs -p)
fi
wait $spawned
rm /$TMPDIR/sentinel.2638[123].log || true
rm ${CONFFILES[@]} || true
return $error_code
}
function write_conf_file() {
local port=$1
local conffile=$(mktemp -t tooz-sentinel-$port-XXXXXX)
cat > $conffile <<EOF
port $port
dir $TMPDIR
sentinel monitor mainbarn 127.0.0.1 6381 2
sentinel down-after-milliseconds mainbarn 80000
sentinel parallel-syncs mainbarn 1
sentinel failover-timeout mainbarn 180000
logfile $TMPDIR/sentinel.$port.log
EOF
echo $conffile
}
trap "clean_exit" EXIT
# If we can't find either redis-server or redis-sentinel, exit.
redis_bin=$(which redis-server)
sentinel_bin=$(which redis-sentinel)
# start redis
$redis_bin --port 6381 &
# start the sentinels
for port in $SENTINEL_PORTS; do
conffile=$(write_conf_file $port)
$sentinel_bin $conffile &
CONFFILES+=($conffile)
done
# Test a first time without sentinel fallbacks
export TOOZ_TEST_REDIS_URL="redis://localhost:26381?sentinel=mainbarn&timeout=5"
$*
# Test a second time with sentinel fallbacks
export TOOZ_TEST_REDIS_URL="redis://localhost:26381?sentinel=mainbarn&sentinel_fallback=localhost:26382&sentinel_fallback=localhost:26383&timeout=5"
$*

View File

@ -1,55 +0,0 @@
#!/bin/bash
set -x -e
ZOO_CONF=/etc/zookeeper
ZOO_BIN=/usr/share/zookeeper/bin
ZOO_TMP_DIR=$(mktemp -d -t ZOO-TMP-XXXXX)
ZOOKEEPER_STARTED=0
mkdir $ZOO_TMP_DIR/bin
function clean_exit(){
local error_code="$?"
if [ -d $ZOO_CONF ] && [ $ZOOKEEPER_STARTED -eq 1 ]; then
stop_zookeeper_server
fi
rm -rf ${ZOO_TMP_DIR}
return $error_code
}
function start_zookeeper_server(){
# Copy zookeeper scripts in temporary directory
cp $ZOO_BIN/* $ZOO_TMP_DIR/bin
# Copy zookeeper conf and set dataDir variable to the zookeeper temporary
# directory
cp $ZOO_CONF/conf/zoo.cfg $ZOO_TMP_DIR
sed -i -r "s@(dataDir *= *).*@\1$ZOO_TMP_DIR@" $ZOO_TMP_DIR/zoo.cfg
# Replace some variables by the zookeeper temporary directory
sed -i -r "s@(ZOOCFGDIR *= *).*@\1$ZOO_TMP_DIR@" $ZOO_TMP_DIR/bin/zkEnv.sh
mkdir $ZOO_TMP_DIR/log
sed -i -r "s@(ZOO_LOG_DIR *= *).*@\1$ZOO_TMP_DIR/log@" $ZOO_TMP_DIR/bin/zkEnv.sh
$ZOO_TMP_DIR/bin/zkServer.sh start
}
function stop_zookeeper_server(){
$ZOO_TMP_DIR/bin/zkServer.sh stop
}
trap "clean_exit" EXIT
if [ -d $ZOO_CONF ]; then
start_zookeeper_server
if [ $? -eq 0 ]; then
ZOOKEEPER_STARTED=1
fi
fi
export TOOZ_TEST_ZOOKEEPER_URL="kazoo://127.0.0.1:2181?timeout=5"
# Yield execution to venv command
$*

View File

@ -16,6 +16,7 @@ testtools>=1.4.0 # MIT
testscenarios>=0.4 # Apache-2.0/BSD
coverage>=3.6 # Apache-2.0
fixtures>=3.0.0 # Apache-2.0/BSD
pifpaf>=0.4.0 # Apache-2.0
# All the various optional drivers require these...
psycopg2>=2.5 # LGPL/ZPL

View File

@ -54,7 +54,7 @@ class TestAPI(testscenarios.TestWithScenarios,
('redis', {'url': os.getenv("TOOZ_TEST_REDIS_URL"),
'bad_url': 'redis://localhost:1',
'timeout_capable': True}),
('postgresql', {'url': os.getenv("TOOZ_TEST_PGSQL_URL"),
('postgresql', {'url': os.getenv("TOOZ_TEST_POSTGRESQL_URL"),
'bad_url': 'postgresql://localhost:1'}),
('mysql', {'url': os.getenv("TOOZ_TEST_MYSQL_URL"),
'bad_url': 'mysql://localhost:1'}),
@ -379,9 +379,17 @@ class TestAPI(testscenarios.TestWithScenarios,
def test_timeout(self):
if not getattr(self, "timeout_capable", False):
self.skipTest("This test only works with timeout capable drivers")
self._coord.stop()
if "?" in self.url:
sep = "&"
else:
sep = "?"
url = self.url + sep + "timeout=5"
self._coord = tooz.coordination.get_coordinator(url, self.member_id)
self._coord.start()
member_id_test2 = self._get_random_uuid()
client2 = tooz.coordination.get_coordinator(self.url,
member_id_test2)
client2 = tooz.coordination.get_coordinator(url, member_id_test2)
client2.start()
self._coord.create_group(self.group_id).get()
self._coord.join_group(self.group_id).get()

View File

@ -15,21 +15,15 @@
# License for the specific language governing permissions and limitations
# under the License.
import os
import uuid
from oslo_utils import encodeutils
import testtools
from testtools import testcase
from tooz import coordination
@testtools.skipUnless(os.getenv("TOOZ_TEST_MYSQL_URL"),
"TOOZ_TEST_MYSQL_URL env variable is not set")
class TestMYSQLDriver(testcase.TestCase):
MYSQL_URL = os.getenv("TOOZ_TEST_MYSQL_URL")
class TestMySQLDriver(testcase.TestCase):
def _create_coordinator(self, url):
@ -48,25 +42,13 @@ class TestMYSQLDriver(testcase.TestCase):
return coord
def test_connect_failure_invalid_hostname_provided(self):
url = self.MYSQL_URL.replace('localhost', 'invalidhost')
c = self._create_coordinator(url)
c = self._create_coordinator("mysql://invalidhost/test")
self.assertRaises(coordination.ToozConnectionError, c.start)
def test_connect_failure_invalid_port_provided(self):
url = self.MYSQL_URL.replace('localhost:13', 'localhost:54')
c = self._create_coordinator(url)
c = self._create_coordinator("mysql://localhost:54/test")
self.assertRaises(coordination.ToozConnectionError, c.start)
def test_connect_failure_invalid_hostname_and_port_provided(self):
url = self.MYSQL_URL.replace('localhost:13', 'invalidhost:54')
c = self._create_coordinator(url)
c = self._create_coordinator("mysql://invalidhost:54/test")
self.assertRaises(coordination.ToozConnectionError, c.start)
def test_connect_failure_invalid_db_name_provided(self):
url = self.MYSQL_URL.replace('test', 'invalid')
c = self._create_coordinator(url)
self.assertRaises(coordination.ToozConnectionError, c.start)
def test_connect_success(self):
c = self._create_coordinator(self.MYSQL_URL)
c.start()

32
tox.ini
View File

@ -15,60 +15,60 @@ commands = python setup.py testr --slowest --testr-args="{posargs}"
doc8 doc/source
[testenv:py27-zookeeper]
commands = {toxinidir}/setup-zookeeper-env.sh python setup.py testr --slowest --testr-args="{posargs}"
commands = pifpaf -e TOOZ_TEST run zookeeper -- python setup.py testr --slowest --testr-args="{posargs}"
[testenv:py34-zookeeper]
basepython = python3.4
commands = {toxinidir}/setup-zookeeper-env.sh python setup.py testr --slowest --testr-args="{posargs}"
commands = pifpaf -e TOOZ_TEST run zookeeper -- python setup.py testr --slowest --testr-args="{posargs}"
[testenv:py27-redis]
commands = {toxinidir}/setup-redis-env.sh python setup.py testr --slowest --testr-args="{posargs}"
commands = pifpaf -e TOOZ_TEST run redis -- python setup.py testr --slowest --testr-args="{posargs}"
[testenv:py34-redis]
basepython = python3.4
commands = {toxinidir}/setup-redis-env.sh python setup.py testr --slowest --testr-args="{posargs}"
commands = pifpaf -e TOOZ_TEST run redis -- python setup.py testr --slowest --testr-args="{posargs}"
[testenv:py27-sentinel]
commands = {toxinidir}/setup-sentinel-env.sh python setup.py testr --slowest --testr-args="{posargs}"
commands = pifpaf -e TOOZ_TEST run redis --sentinel -- python setup.py testr --slowest --testr-args="{posargs}"
[testenv:py34-sentinel]
basepython = python3.4
commands = {toxinidir}/setup-sentinel-env.sh python setup.py testr --slowest --testr-args="{posargs}"
commands = pifpaf -e TOOZ_TEST run redis --sentinel -- python setup.py testr --slowest --testr-args="{posargs}"
[testenv:py27-memcached]
commands = {toxinidir}/setup-memcached-env.sh python setup.py testr --slowest --testr-args="{posargs}"
commands = pifpaf -e TOOZ_TEST run memcached -- python setup.py testr --slowest --testr-args="{posargs}"
[testenv:py34-memcached]
basepython = python3.4
commands = {toxinidir}/setup-memcached-env.sh python setup.py testr --slowest --testr-args="{posargs}"
commands = pifpaf -e TOOZ_TEST run memcached -- python setup.py testr --slowest --testr-args="{posargs}"
[testenv:py27-postgresql]
commands = {toxinidir}/setup-postgresql-env.sh python setup.py testr --slowest --testr-args="{posargs}"
commands = pifpaf -e TOOZ_TEST run postgresql -- python setup.py testr --slowest --testr-args="{posargs}"
[testenv:py34-postgresql]
basepython = python3.4
commands = {toxinidir}/setup-postgresql-env.sh python setup.py testr --slowest --testr-args="{posargs}"
commands = pifpaf -e TOOZ_TEST run postgresql -- python setup.py testr --slowest --testr-args="{posargs}"
[testenv:py27-mysql]
commands = {toxinidir}/setup-mysql-env.sh python setup.py testr --slowest --testr-args="{posargs}"
commands = pifpaf -e TOOZ_TEST run mysql -- python setup.py testr --slowest --testr-args="{posargs}"
[testenv:py34-mysql]
basepython = python3.4
commands = {toxinidir}/setup-mysql-env.sh python setup.py testr --slowest --testr-args="{posargs}"
commands = pifpaf -e TOOZ_TEST run mysql -- python setup.py testr --slowest --testr-args="{posargs}"
[testenv:py27-etcd]
commands = {toxinidir}/setup-etcd-env.sh python setup.py testr --slowest --testr-args="{posargs}"
commands = {toxinidir}/setup-etcd-env.sh pifpaf -g TOOZ_TEST run etcd -- python setup.py testr --slowest --testr-args="{posargs}"
[testenv:py34-etcd]
basepython = python3.4
commands = {toxinidir}/setup-etcd-env.sh python setup.py testr --slowest --testr-args="{posargs}"
commands = {toxinidir}/setup-etcd-env.sh pifpaf -g TOOZ_TEST run etcd -- python setup.py testr --slowest --testr-args="{posargs}"
[testenv:py27-consul]
commands = {toxinidir}/setup-consul-env.sh python setup.py testr --slowest --testr-args="{posargs}"
commands = {toxinidir}/setup-consul-env.sh pifpaf -g TOOZ_TEST run consul -- python setup.py testr --slowest --testr-args="{posargs}"
[testenv:py34-consul]
basepython = python3.4
commands = {toxinidir}/setup-consul-env.sh python setup.py testr --slowest --testr-args="{posargs}"
commands = {toxinidir}/setup-consul-env.sh pifpaf -g TOOZ_TEST run consul -- python setup.py testr --slowest --testr-args="{posargs}"
[testenv:cover]
commands = python setup.py testr --slowest --coverage --testr-args="{posargs}"