diff --git a/setup-consul-env.sh b/setup-consul-env.sh index 144e35d9..21ecef8f 100755 --- a/setup-consul-env.sh +++ b/setup-consul-env.sh @@ -1,108 +1,27 @@ #!/bin/bash +set -eux -set -x -set -e - -CONSUL_DOWN_DIR=`mktemp -d` -CONSUL_BIN_DIR=`mktemp -d` -CONSUL_TMP_DATA_DIR=`mktemp -d` -CONSUL_VERSION="0.6.3" -CONSUL_RELEASE_URL="https://releases.hashicorp.com/consul" - -if [ ! -d "$CONSUL_DOWN_DIR" ]; then - mkdir -p $CONSUL_DOWN_DIR +if [ -z "$(which consul)" ]; then + CONSUL_VERSION=0.6.3 + CONSUL_RELEASE_URL=https://releases.hashicorp.com/consul + case `uname -s` in + Darwin) + consul_file="consul_${CONSUL_VERSION}_darwin_amd64.zip" + ;; + Linux) + consul_file="consul_${CONSUL_VERSION}_linux_amd64.zip" + ;; + *) + echo "Unknown operating system" + exit 1 + ;; + esac + consul_dir=`basename $consul_file .zip` + mkdir -p $consul_dir + curl -L $CONSUL_RELEASE_URL/$CONSUL_VERSION/$consul_file > $consul_dir/$consul_file + unzip $consul_dir/$consul_file -d $consul_dir + export PATH=$PATH:$consul_dir fi -if [ ! -d "$CONSUL_BIN_DIR" ]; then - mkdir -p $CONSUL_BIN_DIR -fi -if [ ! -d "$CONSUL_TMP_DATA_DIR" ]; then - mkdir -p $CONSUL_TMP_DATA_DIR -fi - -function clean_exit(){ - local error_code="$?" - local spawned=$(jobs -p) - if [ -n "$spawned" ]; then - kill $spawned - fi - rm -rf $CONSUL_TMP_DATA_DIR - rm -rf $CONSUL_BIN_DIR - rm -rf $CONSUL_DOWN_DIR - return $error_code -} - -function get_leader(){ - local leader="" - leader=$(curl -s http://127.0.0.1:8500/v1/status/leader) - if [ $? -ne 0 ]; then - return 1 - else - echo $leader | python -c "import sys;\ - import json;\ - print(json.loads(sys.stdin.read()))" - return 0 - fi -} - -function wait_until_up(){ - local leader=`get_leader` - while [ -z "$leader" ]; do - echo "Waiting for consul to respond to a leader http request" - sleep 1 - leader=`get_leader` - done -} - -function download_and_expand_consul() { - if [ "$(uname)" == "Darwin" ]; then - local consul_file="consul_${CONSUL_VERSION}_darwin_amd64.zip" - elif [ "$(uname -a | cut -d" " -f1)" == "Linux" ]; then - local consul_file="consul_${CONSUL_VERSION}_linux_amd64.zip" - else - echo "Unknown operating system '$(uname -a)'" - exit 1 - fi - wget $CONSUL_RELEASE_URL/$CONSUL_VERSION/$consul_file \ - --directory-prefix $CONSUL_DOWN_DIR - if [ $? -ne 0 ]; then - echo "Unable to download consul" - exit 1 - fi - unzip $CONSUL_DOWN_DIR/$consul_file -d $CONSUL_BIN_DIR -} - -function get_consul_version() { - local consul_bin="$1" - local consul_version=`$consul_bin --version | \ - head -n1 | cut -d" " -f2 | \ - cut -d"v" -f2` - echo $consul_version -} - -trap "clean_exit" EXIT - -CONSUL_BIN=`which consul || true` -if [ -z "$CONSUL_BIN" ]; then - echo "Downloading consul $CONSUL_VERSION" - download_and_expand_consul - CONSUL_BIN=$CONSUL_BIN_DIR/consul - if [ ! -e "$CONSUL_BIN" ]; then - echo "Consul executable does not exist (even after downloading it)" - exit 1 - fi -else - CONSUL_VERSION=`get_consul_version "$CONSUL_BIN"` - echo "Consul $CONSUL_VERSION is already installed" -fi - -$CONSUL_BIN agent -server -bootstrap-expect 1 -data-dir $CONSUL_TMP_DATA_DIR -node=agent-one -bind=127.0.0.1 & - -# Give some time for the agent to elect a leader, and startup... -# TODO(harlowja): there has to be a better way to do this, there doesn't -# seem -wait_until_up - -export TOOZ_TEST_CONSUL_URL="consul://localhost:8500/v1" # Yield execution to venv command $* diff --git a/setup-etcd-env.sh b/setup-etcd-env.sh index d27fe8c5..34fdf2a5 100755 --- a/setup-etcd-env.sh +++ b/setup-etcd-env.sh @@ -1,16 +1,6 @@ #!/bin/bash set -eux - -clean_exit() { - local error_code="$?" - kill $(jobs -p) - return $error_code -} - -trap clean_exit EXIT -if [ -n "$(which etcd)" ]; then - etcd & -else +if [ -z "$(which etcd)" ]; then ETCD_VERSION=2.2.2 case `uname -s` in Darwin) @@ -35,9 +25,7 @@ else esac TARBALL_NAME=etcd-v${ETCD_VERSION}-$OS-$MACHINE test ! -d "$TARBALL_NAME" && curl -L https://github.com/coreos/etcd/releases/download/v${ETCD_VERSION}/${TARBALL_NAME}.${SUFFIX} | tar xz - $TARBALL_NAME/etcd & + export PATH=$PATH:$TARBALL_NAME fi -export TOOZ_TEST_ETCD_URL="etcd://localhost:4001" -# Yield execution to venv command $* diff --git a/setup-memcached-env.sh b/setup-memcached-env.sh deleted file mode 100755 index b71c7a05..00000000 --- a/setup-memcached-env.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -set -x -e - -function clean_exit(){ - local error_code="$?" - local spawned=$(jobs -p) - if [ -n "$spawned" ]; then - kill $(jobs -p) - fi - return $error_code -} - -trap "clean_exit" EXIT - -memcached_bin=$(which memcached || true) -if [ -n "$memcached_bin" ]; then - $memcached_bin -p 11212 & -fi - -export TOOZ_TEST_MEMCACHED_URL="memcached://localhost:11212?timeout=5" -# Yield execution to venv command -$* diff --git a/setup-mysql-env.sh b/setup-mysql-env.sh deleted file mode 100755 index 77f014fc..00000000 --- a/setup-mysql-env.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -set -x -e - -clean_exit () { - local error_code="$?" - kill $(jobs -p) - rm -rf ${MYSQL_DATA} - return $error_code -} - -wait_for_line () { - while read line - do - echo "$line" | grep -q "$1" && break - done < "$2" - # Read the fifo for ever otherwise process would block - cat "$2" >/dev/null & -} - -get_random_port () { - PORT=13306 - while netstat -atwn | grep "^.*:${PORT}.*:\*\s*LISTEN\s*$" - do - PORT=$(( ${PORT} + 1 )) - done -} - -trap "clean_exit" EXIT - -# On systems like Fedora here's where mysqld can be found -export PATH=$PATH:/usr/libexec - -# Start MySQL process for tests -MYSQL_DATA=`mktemp -d -t tooz-mysql-XXXXX` -mkfifo ${MYSQL_DATA}/out -# Initialize MySQL Data Directory -mysql_install_db --user=${USER} --ldata=${MYSQL_DATA} -# Get random unused port for mysql -get_random_port -# Start mysqld with networking (i.e. - allow connection with TCP/IP) -mysqld --no-defaults --datadir=${MYSQL_DATA} --port=${PORT} --pid-file=${MYSQL_DATA}/mysql.pid --socket=${MYSQL_DATA}/mysql.socket --slow-query-log-file=${MYSQL_DATA}/mysql-slow.log --skip-grant-tables &> ${MYSQL_DATA}/out & -# Wait for MySQL to start listening to connections -wait_for_line "mysqld: ready for connections." ${MYSQL_DATA}/out -# The default root password is blank -mysql --host=localhost -S ${MYSQL_DATA}/mysql.socket -e 'CREATE DATABASE IF NOT EXISTS test;' -export TOOZ_TEST_MYSQL_URL="mysql://root@localhost:${PORT}/test" - -# Yield execution to venv command -$* diff --git a/setup-postgresql-env.sh b/setup-postgresql-env.sh deleted file mode 100755 index 9d1ec126..00000000 --- a/setup-postgresql-env.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -set -x -e - -function clean_exit() { - local error_code="$?" - ${PGSQL_PATH}/pg_ctl -w -D ${PGSQL_DATA} -o "-p $PGSQL_PORT" stop - rm -rf ${PGSQL_DATA} - return $error_code -} - -trap "clean_exit" EXIT - -# Start PostgreSQL process for tests -PGSQL_DATA=`mktemp -d -t tooz-pgsql-XXXXX` -PGSQL_PATH=`pg_config --bindir` -PGSQL_PORT=9825 -${PGSQL_PATH}/initdb ${PGSQL_DATA} -${PGSQL_PATH}/pg_ctl -w -D ${PGSQL_DATA} -o "-k ${PGSQL_DATA} -p ${PGSQL_PORT}" start -# Wait for PostgreSQL to start listening to connections -export TOOZ_TEST_PGSQL_URL="postgresql:///?host=${PGSQL_DATA}&port=${PGSQL_PORT}&dbname=template1" - -# Yield execution to venv command -$* diff --git a/setup-redis-env.sh b/setup-redis-env.sh deleted file mode 100755 index fe068da7..00000000 --- a/setup-redis-env.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -set -x -e - -function clean_exit(){ - local error_code="$?" - local spawned=$(jobs -p) - if [ -n "$spawned" ]; then - kill $(jobs -p) - fi - return $error_code -} - -trap "clean_exit" EXIT - -redis_bin=$(which redis-server || true) -if [ -n "$redis_bin" ]; then - $redis_bin --port 6380 & -fi - -export TOOZ_TEST_REDIS_URL="redis://localhost:6380?timeout=5" -# Yield execution to venv command -$* diff --git a/setup-sentinel-env.sh b/setup-sentinel-env.sh deleted file mode 100755 index 8aa0c291..00000000 --- a/setup-sentinel-env.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -set -x -e - -SENTINEL_PORTS="26381 26382 26383" -CONFFILES=() -TMPDIR=${TMPDIR:/tmp} - -function clean_exit(){ - local error_code="$?" - local spawned=$(jobs -p) - if [ -n "$spawned" ]; then - kill $(jobs -p) - fi - wait $spawned - rm /$TMPDIR/sentinel.2638[123].log || true - rm ${CONFFILES[@]} || true - return $error_code -} - -function write_conf_file() { - local port=$1 - local conffile=$(mktemp -t tooz-sentinel-$port-XXXXXX) - cat > $conffile <=1.4.0 # MIT testscenarios>=0.4 # Apache-2.0/BSD coverage>=3.6 # Apache-2.0 fixtures>=3.0.0 # Apache-2.0/BSD +pifpaf>=0.4.0 # Apache-2.0 # All the various optional drivers require these... psycopg2>=2.5 # LGPL/ZPL diff --git a/tooz/tests/test_coordination.py b/tooz/tests/test_coordination.py index 6728ec69..447249be 100644 --- a/tooz/tests/test_coordination.py +++ b/tooz/tests/test_coordination.py @@ -54,7 +54,7 @@ class TestAPI(testscenarios.TestWithScenarios, ('redis', {'url': os.getenv("TOOZ_TEST_REDIS_URL"), 'bad_url': 'redis://localhost:1', 'timeout_capable': True}), - ('postgresql', {'url': os.getenv("TOOZ_TEST_PGSQL_URL"), + ('postgresql', {'url': os.getenv("TOOZ_TEST_POSTGRESQL_URL"), 'bad_url': 'postgresql://localhost:1'}), ('mysql', {'url': os.getenv("TOOZ_TEST_MYSQL_URL"), 'bad_url': 'mysql://localhost:1'}), @@ -379,9 +379,17 @@ class TestAPI(testscenarios.TestWithScenarios, def test_timeout(self): if not getattr(self, "timeout_capable", False): self.skipTest("This test only works with timeout capable drivers") + self._coord.stop() + if "?" in self.url: + sep = "&" + else: + sep = "?" + url = self.url + sep + "timeout=5" + self._coord = tooz.coordination.get_coordinator(url, self.member_id) + self._coord.start() + member_id_test2 = self._get_random_uuid() - client2 = tooz.coordination.get_coordinator(self.url, - member_id_test2) + client2 = tooz.coordination.get_coordinator(url, member_id_test2) client2.start() self._coord.create_group(self.group_id).get() self._coord.join_group(self.group_id).get() diff --git a/tooz/tests/test_mysql.py b/tooz/tests/test_mysql.py index 6b0176d6..b8b3b5db 100644 --- a/tooz/tests/test_mysql.py +++ b/tooz/tests/test_mysql.py @@ -15,21 +15,15 @@ # License for the specific language governing permissions and limitations # under the License. -import os import uuid from oslo_utils import encodeutils -import testtools from testtools import testcase from tooz import coordination -@testtools.skipUnless(os.getenv("TOOZ_TEST_MYSQL_URL"), - "TOOZ_TEST_MYSQL_URL env variable is not set") -class TestMYSQLDriver(testcase.TestCase): - - MYSQL_URL = os.getenv("TOOZ_TEST_MYSQL_URL") +class TestMySQLDriver(testcase.TestCase): def _create_coordinator(self, url): @@ -48,25 +42,13 @@ class TestMYSQLDriver(testcase.TestCase): return coord def test_connect_failure_invalid_hostname_provided(self): - url = self.MYSQL_URL.replace('localhost', 'invalidhost') - c = self._create_coordinator(url) + c = self._create_coordinator("mysql://invalidhost/test") self.assertRaises(coordination.ToozConnectionError, c.start) def test_connect_failure_invalid_port_provided(self): - url = self.MYSQL_URL.replace('localhost:13', 'localhost:54') - c = self._create_coordinator(url) + c = self._create_coordinator("mysql://localhost:54/test") self.assertRaises(coordination.ToozConnectionError, c.start) def test_connect_failure_invalid_hostname_and_port_provided(self): - url = self.MYSQL_URL.replace('localhost:13', 'invalidhost:54') - c = self._create_coordinator(url) + c = self._create_coordinator("mysql://invalidhost:54/test") self.assertRaises(coordination.ToozConnectionError, c.start) - - def test_connect_failure_invalid_db_name_provided(self): - url = self.MYSQL_URL.replace('test', 'invalid') - c = self._create_coordinator(url) - self.assertRaises(coordination.ToozConnectionError, c.start) - - def test_connect_success(self): - c = self._create_coordinator(self.MYSQL_URL) - c.start() diff --git a/tox.ini b/tox.ini index ecc463b6..33b2eb6f 100644 --- a/tox.ini +++ b/tox.ini @@ -15,60 +15,60 @@ commands = python setup.py testr --slowest --testr-args="{posargs}" doc8 doc/source [testenv:py27-zookeeper] -commands = {toxinidir}/setup-zookeeper-env.sh python setup.py testr --slowest --testr-args="{posargs}" +commands = pifpaf -e TOOZ_TEST run zookeeper -- python setup.py testr --slowest --testr-args="{posargs}" [testenv:py34-zookeeper] basepython = python3.4 -commands = {toxinidir}/setup-zookeeper-env.sh python setup.py testr --slowest --testr-args="{posargs}" +commands = pifpaf -e TOOZ_TEST run zookeeper -- python setup.py testr --slowest --testr-args="{posargs}" [testenv:py27-redis] -commands = {toxinidir}/setup-redis-env.sh python setup.py testr --slowest --testr-args="{posargs}" +commands = pifpaf -e TOOZ_TEST run redis -- python setup.py testr --slowest --testr-args="{posargs}" [testenv:py34-redis] basepython = python3.4 -commands = {toxinidir}/setup-redis-env.sh python setup.py testr --slowest --testr-args="{posargs}" +commands = pifpaf -e TOOZ_TEST run redis -- python setup.py testr --slowest --testr-args="{posargs}" [testenv:py27-sentinel] -commands = {toxinidir}/setup-sentinel-env.sh python setup.py testr --slowest --testr-args="{posargs}" +commands = pifpaf -e TOOZ_TEST run redis --sentinel -- python setup.py testr --slowest --testr-args="{posargs}" [testenv:py34-sentinel] basepython = python3.4 -commands = {toxinidir}/setup-sentinel-env.sh python setup.py testr --slowest --testr-args="{posargs}" +commands = pifpaf -e TOOZ_TEST run redis --sentinel -- python setup.py testr --slowest --testr-args="{posargs}" [testenv:py27-memcached] -commands = {toxinidir}/setup-memcached-env.sh python setup.py testr --slowest --testr-args="{posargs}" +commands = pifpaf -e TOOZ_TEST run memcached -- python setup.py testr --slowest --testr-args="{posargs}" [testenv:py34-memcached] basepython = python3.4 -commands = {toxinidir}/setup-memcached-env.sh python setup.py testr --slowest --testr-args="{posargs}" +commands = pifpaf -e TOOZ_TEST run memcached -- python setup.py testr --slowest --testr-args="{posargs}" [testenv:py27-postgresql] -commands = {toxinidir}/setup-postgresql-env.sh python setup.py testr --slowest --testr-args="{posargs}" +commands = pifpaf -e TOOZ_TEST run postgresql -- python setup.py testr --slowest --testr-args="{posargs}" [testenv:py34-postgresql] basepython = python3.4 -commands = {toxinidir}/setup-postgresql-env.sh python setup.py testr --slowest --testr-args="{posargs}" +commands = pifpaf -e TOOZ_TEST run postgresql -- python setup.py testr --slowest --testr-args="{posargs}" [testenv:py27-mysql] -commands = {toxinidir}/setup-mysql-env.sh python setup.py testr --slowest --testr-args="{posargs}" +commands = pifpaf -e TOOZ_TEST run mysql -- python setup.py testr --slowest --testr-args="{posargs}" [testenv:py34-mysql] basepython = python3.4 -commands = {toxinidir}/setup-mysql-env.sh python setup.py testr --slowest --testr-args="{posargs}" +commands = pifpaf -e TOOZ_TEST run mysql -- python setup.py testr --slowest --testr-args="{posargs}" [testenv:py27-etcd] -commands = {toxinidir}/setup-etcd-env.sh python setup.py testr --slowest --testr-args="{posargs}" +commands = {toxinidir}/setup-etcd-env.sh pifpaf -g TOOZ_TEST run etcd -- python setup.py testr --slowest --testr-args="{posargs}" [testenv:py34-etcd] basepython = python3.4 -commands = {toxinidir}/setup-etcd-env.sh python setup.py testr --slowest --testr-args="{posargs}" +commands = {toxinidir}/setup-etcd-env.sh pifpaf -g TOOZ_TEST run etcd -- python setup.py testr --slowest --testr-args="{posargs}" [testenv:py27-consul] -commands = {toxinidir}/setup-consul-env.sh python setup.py testr --slowest --testr-args="{posargs}" +commands = {toxinidir}/setup-consul-env.sh pifpaf -g TOOZ_TEST run consul -- python setup.py testr --slowest --testr-args="{posargs}" [testenv:py34-consul] basepython = python3.4 -commands = {toxinidir}/setup-consul-env.sh python setup.py testr --slowest --testr-args="{posargs}" +commands = {toxinidir}/setup-consul-env.sh pifpaf -g TOOZ_TEST run consul -- python setup.py testr --slowest --testr-args="{posargs}" [testenv:cover] commands = python setup.py testr --slowest --coverage --testr-args="{posargs}"