Merge "Test using mysql + postgres if available"

This commit is contained in:
Jenkins
2014-01-27 09:52:45 +00:00
committed by Gerrit Code Review
4 changed files with 411 additions and 120 deletions

View File

@@ -16,41 +16,248 @@
# License for the specific language governing permissions and limitations
# under the License.
import contextlib
import os
import tempfile
import threading
import testtools
# NOTE(harlowja): by default this will test against sqlite using a temporary
# sqlite file (this is done instead of in-memory to ensure thread safety,
# in-memory sqlite is not thread safe).
#
# There are also "opportunistic" tests for both mysql and postgresql in here,
# which allows testing against all 3 databases (sqlite, mysql, postgres) in
# a properly configured unit test environment. For the opportunistic testing
# you need to set up a db named 'openstack_citest' with user 'openstack_citest'
# and password 'openstack_citest' on localhost.
USER = "openstack_citest"
PASSWD = "openstack_citest"
DATABASE = "openstack_citest"
try:
from taskflow.persistence.backends import impl_sqlalchemy
import sqlalchemy as sa
from sqlalchemy import exc as sa_exc
SQLALCHEMY_AVAILABLE = True
except ImportError:
SQLALCHEMY_AVAILABLE = False
# Testing will try to run against these two mysql library variants.
MYSQL_VARIANTS = ('mysqldb', 'pymysql')
from taskflow import test
from taskflow.tests.unit.persistence import base
from taskflow.utils import lock_utils
def _get_connect_string(backend, user, passwd, database=None, variant=None):
"""Try to get a connection with a very specific set of values, if we get
these then we'll run the tests, otherwise they are skipped.
"""
if backend == "postgres":
if not variant:
variant = 'psycopg2'
backend = "postgresql+%s" % (variant)
elif backend == "mysql":
if not variant:
variant = 'mysqldb'
backend = "mysql+%s" % (variant)
else:
raise Exception("Unrecognized backend: '%s'" % backend)
if not database:
database = ''
return "%s://%s:%s@localhost/%s" % (backend, user, passwd, database)
def _mysql_exists():
if not SQLALCHEMY_AVAILABLE:
return False
for variant in MYSQL_VARIANTS:
engine = None
db_uri = _get_connect_string('mysql', USER, PASSWD, variant=variant)
try:
engine = sa.create_engine(db_uri)
with contextlib.closing(engine.connect()):
return True
except (sa_exc.DatabaseError, ImportError):
pass
finally:
if engine is not None:
engine.dispose()
return False
def _postgres_exists():
if not SQLALCHEMY_AVAILABLE:
return False
engine = None
connection_uri = _get_connect_string('postgres', USER, PASSWD, 'template1')
try:
engine = sa.create_engine(connection_uri)
with contextlib.closing(engine.connect()):
return True
except (sa_exc.DatabaseError, ImportError):
return False
finally:
if engine is not None:
engine.dispose()
@testtools.skipIf(not SQLALCHEMY_AVAILABLE, 'sqlalchemy is not available')
class SqlPersistenceTest(test.TestCase, base.PersistenceTestMixin):
class SqlitePersistenceTest(test.TestCase, base.PersistenceTestMixin):
"""Inherits from the base test and sets up a sqlite temporary db."""
def _get_connection(self):
conf = {
'connection': self.db_uri,
}
conn = impl_sqlalchemy.SQLAlchemyBackend(conf).get_connection()
return conn
return impl_sqlalchemy.SQLAlchemyBackend(conf).get_connection()
def setUp(self):
super(SqlPersistenceTest, self).setUp()
super(SqlitePersistenceTest, self).setUp()
self.db_location = tempfile.mktemp(suffix='.db')
self.db_uri = "sqlite:///%s" % (self.db_location)
# Ensure upgraded to the right schema
conn = self._get_connection()
conn.upgrade()
with contextlib.closing(self._get_connection()) as conn:
conn.upgrade()
def tearDown(self):
super(SqlPersistenceTest, self).tearDown()
super(SqlitePersistenceTest, self).tearDown()
if self.db_location and os.path.isfile(self.db_location):
os.unlink(self.db_location)
self.db_location = None
class BackendPersistenceTestMixin(base.PersistenceTestMixin):
"""Specifies a backend type and does required setup and teardown."""
LOCK_NAME = None
def _get_connection(self):
return self.backend.get_connection()
def _reset_database(self):
"""Resets the database, and returns the uri to that database.
Called *only* after locking succeeds.
"""
raise NotImplementedError()
def tearDown(self):
super(BackendPersistenceTestMixin, self).tearDown()
try:
if self.backend is not None:
self.backend.close()
self.backend = None
finally:
self.big_lock.release()
def setUp(self):
super(BackendPersistenceTestMixin, self).setUp()
self.backend = None
self.big_lock.acquire()
conf = {
'connection': self._reset_database(),
}
# Ensure upgraded to the right schema
self.backend = impl_sqlalchemy.SQLAlchemyBackend(conf)
with contextlib.closing(self._get_connection()) as conn:
conn.upgrade()
@testtools.skipIf(not SQLALCHEMY_AVAILABLE, 'sqlalchemy is not available')
@testtools.skipIf(not _mysql_exists(), 'mysql is not available')
class MysqlPersistenceTest(BackendPersistenceTestMixin, test.TestCase):
LOCK_NAME = 'mysql_persistence_test'
def __init__(self, *args, **kwargs):
test.TestCase.__init__(self, *args, **kwargs)
# We need to make sure that each test goes through a set of locks
# to ensure that multiple tests are not modifying the database,
# dropping it, creating it at the same time. To accomplish this we use
# a lock that ensures multiple parallel processes can't run at the
# same time as well as a in-process lock to ensure that multiple
# threads can't run at the same time.
lock_path = os.path.join(tempfile.gettempdir(),
'taskflow-%s.lock' % (self.LOCK_NAME))
locks = [
lock_utils.InterProcessLock(lock_path),
threading.RLock(),
]
self.big_lock = lock_utils.MultiLock(locks)
def _reset_database(self):
working_variant = None
for variant in MYSQL_VARIANTS:
engine = None
try:
db_uri = _get_connect_string('mysql', USER, PASSWD,
variant=variant)
engine = sa.create_engine(db_uri)
with contextlib.closing(engine.connect()) as conn:
conn.execute("DROP DATABASE IF EXISTS %s" % DATABASE)
conn.execute("CREATE DATABASE %s" % DATABASE)
working_variant = variant
except (sa_exc.DatabaseError, ImportError):
pass
finally:
if engine is not None:
engine.dispose()
if working_variant:
break
if not working_variant:
variants = ", ".join(MYSQL_VARIANTS)
raise self.skipException("Failed to find a mysql variant"
" (tried %s) that works; mysql testing"
" being skipped" % (variants))
else:
return _get_connect_string('mysql', USER, PASSWD,
database=DATABASE,
variant=working_variant)
@testtools.skipIf(not SQLALCHEMY_AVAILABLE, 'sqlalchemy is not available')
@testtools.skipIf(not _postgres_exists(), 'postgres is not available')
class PostgresPersistenceTest(BackendPersistenceTestMixin, test.TestCase):
LOCK_NAME = 'postgres_persistence_test'
def __init__(self, *args, **kwargs):
test.TestCase.__init__(self, *args, **kwargs)
# We need to make sure that each test goes through a set of locks
# to ensure that multiple tests are not modifying the database,
# dropping it, creating it at the same time. To accomplish this we use
# a lock that ensures multiple parallel processes can't run at the
# same time as well as a in-process lock to ensure that multiple
# threads can't run at the same time.
lock_path = os.path.join(tempfile.gettempdir(),
'taskflow-%s.lock' % (self.LOCK_NAME))
locks = [
lock_utils.InterProcessLock(lock_path),
threading.RLock(),
]
self.big_lock = lock_utils.MultiLock(locks)
def _reset_database(self):
engine = None
try:
# Postgres can't operate on the database its connected to, thats
# why we connect to the default template database 'template1' and
# then drop and create the desired database.
tmp_uri = _get_connect_string('postgres', USER, PASSWD,
database='template1')
engine = sa.create_engine(tmp_uri)
with contextlib.closing(engine.connect()) as conn:
conn.connection.set_isolation_level(0)
conn.execute("DROP DATABASE IF EXISTS %s" % DATABASE)
conn.connection.set_isolation_level(1)
with contextlib.closing(engine.connect()) as conn:
conn.connection.set_isolation_level(0)
conn.execute("CREATE DATABASE %s" % DATABASE)
conn.connection.set_isolation_level(1)
finally:
if engine is not None:
engine.dispose()
return _get_connect_string('postgres', USER, PASSWD, database=DATABASE)

View File

@@ -77,6 +77,9 @@ class MultiLock(object):
self._locked = [False] * len(locks)
def __enter__(self):
self.acquire()
def acquire(self):
def is_locked(lock):
# NOTE(harlowja): reentrant locks (rlock) don't have this
@@ -90,10 +93,14 @@ class MultiLock(object):
raise threading.ThreadError("Lock %s not previously released"
% (i + 1))
self._locked[i] = False
for (i, lock) in enumerate(self._locks):
self._locked[i] = lock.acquire()
def __exit__(self, type, value, traceback):
self.release()
def release(self):
for (i, locked) in enumerate(self._locked):
try:
if locked:
@@ -127,9 +134,16 @@ class _InterProcessLock(object):
def path(self):
return self._fname
def __enter__(self):
self._lockfile = open(self.path, 'w')
def release(self):
try:
self.unlock()
self._lockfile.close()
except IOError:
LOG.exception("Could not release the acquired lock `%s`",
self.path)
def acquire(self):
self._lockfile = open(self.path, 'w')
while True:
try:
# Using non-blocking locks since green threads are not
@@ -137,20 +151,19 @@ class _InterProcessLock(object):
# Also upon reading the MSDN docs for locking(), it seems
# to have a laughable 10 attempts "blocking" mechanism.
self.trylock()
return self
return True
except IOError as e:
if e.errno in (errno.EACCES, errno.EAGAIN):
time.sleep(WAIT_TIME)
else:
raise
def __enter__(self):
self.acquire()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
try:
self.unlock()
self._lockfile.close()
except IOError:
LOG.exception("Could not release the acquired lock `%s`",
self.path)
self.release()
def trylock(self):
raise NotImplementedError()

View File

@@ -15,6 +15,8 @@ setenv = VIRTUAL_ENV={envdir}
LC_ALL=C
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
alembic>=0.4.1
psycopg2
commands = python setup.py testr --slowest --testr-args='{posargs}'
[tox:jenkins]
@@ -31,7 +33,7 @@ commands = pylint
[testenv:cover]
basepython = python2.7
deps = {[testenv:py27-sa9-ev]deps}
deps = {[testenv:py27-sa9-mysql-ev]deps}
commands = python setup.py testr --coverage --testr-args='{posargs}'
[testenv:venv]
@@ -45,19 +47,20 @@ exclude = .venv,.tox,dist,doc,./taskflow/openstack/common,*egg,.git,build,tools
# richest set of test requirements
[testenv:py26]
basepython = python2.6
deps = {[testenv:py27-sa7-ev]deps}
deps = {[testenv:py26-sa7-mysql-ev]deps}
[testenv:py27]
basepython = python2.7
deps = {[testenv:py27-sa9-ev]deps}
deps = {[testenv:py27-sa9-mysql-ev]deps}
[testenv:py33]
basepython = python3.3
deps = {[testenv:py33-sa9]deps}
deps = {[testenv:py33-sa9-pymysql]deps}
[axes]
python = py26,py27,py33
sqlalchemy = sa7,sa8,sa9,nosa
sqlalchemy = sa7,sa8,sa9
mysql = mysql,pymysql
eventlet = ev,*
[axis:python:py26]
@@ -75,23 +78,26 @@ deps = {[testenv]deps}
[axis:eventlet:ev]
deps =
eventlet>=0.13.0
constraints=
constraints =
!python:py33
[axis:sqlalchemy:sa7]
deps =
SQLAlchemy<=0.7.99
alembic>=0.4.1
[axis:sqlalchemy:sa8]
deps =
SQLAlchemy>=0.8,<=0.8.99
alembic>=0.4.1
[axis:sqlalchemy:sa9]
deps =
SQLAlchemy>=0.9,<=0.9.99
alembic>=0.4.1
[axis:sqlalchemy:nosa]
# empty
[axis:mysql:mysql]
deps =
MySQL-python
constraints = !python:py33
[axis:mysql:pymysql]
deps =
pyMySQL

251
tox.ini
View File

@@ -6,28 +6,35 @@ skipsdist = True
envlist = cover,
pep8,
py26,
py26-nosa,
py26-nosa-ev,
py26-sa7,
py26-sa7-ev,
py26-sa8,
py26-sa8-ev,
py26-sa9,
py26-sa9-ev,
py26-sa7-mysql,
py26-sa7-mysql-ev,
py26-sa7-pymysql,
py26-sa7-pymysql-ev,
py26-sa8-mysql,
py26-sa8-mysql-ev,
py26-sa8-pymysql,
py26-sa8-pymysql-ev,
py26-sa9-mysql,
py26-sa9-mysql-ev,
py26-sa9-pymysql,
py26-sa9-pymysql-ev,
py27,
py27-nosa,
py27-nosa-ev,
py27-sa7,
py27-sa7-ev,
py27-sa8,
py27-sa8-ev,
py27-sa9,
py27-sa9-ev,
py27-sa7-mysql,
py27-sa7-mysql-ev,
py27-sa7-pymysql,
py27-sa7-pymysql-ev,
py27-sa8-mysql,
py27-sa8-mysql-ev,
py27-sa8-pymysql,
py27-sa8-pymysql-ev,
py27-sa9-mysql,
py27-sa9-mysql-ev,
py27-sa9-pymysql,
py27-sa9-pymysql-ev,
py33,
py33-nosa,
py33-sa7,
py33-sa8,
py33-sa9,
py33-sa7-pymysql,
py33-sa8-pymysql,
py33-sa9-pymysql,
pylint
[testenv]
@@ -39,6 +46,8 @@ setenv = VIRTUAL_ENV={envdir}
LC_ALL=C
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
alembic>=0.4.1
psycopg2
commands = python setup.py testr --slowest --testr-args='{posargs}'
[tox:jenkins]
@@ -55,7 +64,7 @@ commands = pylint
[testenv:cover]
basepython = python2.7
deps = {[testenv:py27-sa9-ev]deps}
deps = {[testenv:py27-sa9-mysql-ev]deps}
commands = python setup.py testr --coverage --testr-args='{posargs}'
[testenv:venv]
@@ -67,131 +76,187 @@ exclude = .venv,.tox,dist,doc,./taskflow/openstack/common,*egg,.git,build,tools
[testenv:py26]
basepython = python2.6
deps = {[testenv:py27-sa7-ev]deps}
deps = {[testenv:py26-sa7-mysql-ev]deps}
[testenv:py27]
basepython = python2.7
deps = {[testenv:py27-sa9-ev]deps}
deps = {[testenv:py27-sa9-mysql-ev]deps}
[testenv:py33]
basepython = python3.3
deps = {[testenv:py33-sa9]deps}
deps = {[testenv:py33-sa9-pymysql]deps}
[testenv:py26-sa7-ev]
[testenv:py26-sa7-mysql-ev]
deps = {[testenv]deps}
SQLAlchemy<=0.7.99
alembic>=0.4.1
MySQL-python
eventlet>=0.13.0
basepython = python2.6
[testenv:py26-sa7]
[testenv:py26-sa7-mysql]
deps = {[testenv]deps}
SQLAlchemy<=0.7.99
alembic>=0.4.1
MySQL-python
basepython = python2.6
[testenv:py26-sa8-ev]
[testenv:py26-sa7-pymysql-ev]
deps = {[testenv]deps}
SQLAlchemy<=0.7.99
pyMySQL
eventlet>=0.13.0
basepython = python2.6
[testenv:py26-sa7-pymysql]
deps = {[testenv]deps}
SQLAlchemy<=0.7.99
pyMySQL
basepython = python2.6
[testenv:py26-sa8-mysql-ev]
deps = {[testenv]deps}
SQLAlchemy>=0.8,<=0.8.99
alembic>=0.4.1
MySQL-python
eventlet>=0.13.0
basepython = python2.6
[testenv:py26-sa8]
[testenv:py26-sa8-mysql]
deps = {[testenv]deps}
SQLAlchemy>=0.8,<=0.8.99
alembic>=0.4.1
MySQL-python
basepython = python2.6
[testenv:py26-sa9-ev]
deps = {[testenv]deps}
SQLAlchemy>=0.9,<=0.9.99
alembic>=0.4.1
eventlet>=0.13.0
basepython = python2.6
[testenv:py26-sa9]
deps = {[testenv]deps}
SQLAlchemy>=0.9,<=0.9.99
alembic>=0.4.1
basepython = python2.6
[testenv:py26-nosa-ev]
deps = {[testenv]deps}
eventlet>=0.13.0
basepython = python2.6
[testenv:py26-nosa]
deps = {[testenv]deps}
basepython = python2.6
[testenv:py27-sa7-ev]
deps = {[testenv]deps}
SQLAlchemy<=0.7.99
alembic>=0.4.1
eventlet>=0.13.0
basepython = python2.7
[testenv:py27-sa7]
deps = {[testenv]deps}
SQLAlchemy<=0.7.99
alembic>=0.4.1
basepython = python2.7
[testenv:py27-sa8-ev]
[testenv:py26-sa8-pymysql-ev]
deps = {[testenv]deps}
SQLAlchemy>=0.8,<=0.8.99
alembic>=0.4.1
pyMySQL
eventlet>=0.13.0
basepython = python2.7
basepython = python2.6
[testenv:py27-sa8]
[testenv:py26-sa8-pymysql]
deps = {[testenv]deps}
SQLAlchemy>=0.8,<=0.8.99
alembic>=0.4.1
basepython = python2.7
pyMySQL
basepython = python2.6
[testenv:py27-sa9-ev]
[testenv:py26-sa9-mysql-ev]
deps = {[testenv]deps}
SQLAlchemy>=0.9,<=0.9.99
alembic>=0.4.1
MySQL-python
eventlet>=0.13.0
basepython = python2.7
basepython = python2.6
[testenv:py27-sa9]
[testenv:py26-sa9-mysql]
deps = {[testenv]deps}
SQLAlchemy>=0.9,<=0.9.99
alembic>=0.4.1
basepython = python2.7
MySQL-python
basepython = python2.6
[testenv:py27-nosa-ev]
[testenv:py26-sa9-pymysql-ev]
deps = {[testenv]deps}
SQLAlchemy>=0.9,<=0.9.99
pyMySQL
eventlet>=0.13.0
basepython = python2.7
basepython = python2.6
[testenv:py27-nosa]
[testenv:py26-sa9-pymysql]
deps = {[testenv]deps}
basepython = python2.7
SQLAlchemy>=0.9,<=0.9.99
pyMySQL
basepython = python2.6
[testenv:py33-sa7]
[testenv:py27-sa7-mysql-ev]
deps = {[testenv]deps}
SQLAlchemy<=0.7.99
alembic>=0.4.1
MySQL-python
eventlet>=0.13.0
basepython = python2.7
[testenv:py27-sa7-mysql]
deps = {[testenv]deps}
SQLAlchemy<=0.7.99
MySQL-python
basepython = python2.7
[testenv:py27-sa7-pymysql-ev]
deps = {[testenv]deps}
SQLAlchemy<=0.7.99
pyMySQL
eventlet>=0.13.0
basepython = python2.7
[testenv:py27-sa7-pymysql]
deps = {[testenv]deps}
SQLAlchemy<=0.7.99
pyMySQL
basepython = python2.7
[testenv:py27-sa8-mysql-ev]
deps = {[testenv]deps}
SQLAlchemy>=0.8,<=0.8.99
MySQL-python
eventlet>=0.13.0
basepython = python2.7
[testenv:py27-sa8-mysql]
deps = {[testenv]deps}
SQLAlchemy>=0.8,<=0.8.99
MySQL-python
basepython = python2.7
[testenv:py27-sa8-pymysql-ev]
deps = {[testenv]deps}
SQLAlchemy>=0.8,<=0.8.99
pyMySQL
eventlet>=0.13.0
basepython = python2.7
[testenv:py27-sa8-pymysql]
deps = {[testenv]deps}
SQLAlchemy>=0.8,<=0.8.99
pyMySQL
basepython = python2.7
[testenv:py27-sa9-mysql-ev]
deps = {[testenv]deps}
SQLAlchemy>=0.9,<=0.9.99
MySQL-python
eventlet>=0.13.0
basepython = python2.7
[testenv:py27-sa9-mysql]
deps = {[testenv]deps}
SQLAlchemy>=0.9,<=0.9.99
MySQL-python
basepython = python2.7
[testenv:py27-sa9-pymysql-ev]
deps = {[testenv]deps}
SQLAlchemy>=0.9,<=0.9.99
pyMySQL
eventlet>=0.13.0
basepython = python2.7
[testenv:py27-sa9-pymysql]
deps = {[testenv]deps}
SQLAlchemy>=0.9,<=0.9.99
pyMySQL
basepython = python2.7
[testenv:py33-sa7-pymysql]
deps = {[testenv]deps}
SQLAlchemy<=0.7.99
pyMySQL
basepython = python3.3
[testenv:py33-sa8]
[testenv:py33-sa8-pymysql]
deps = {[testenv]deps}
SQLAlchemy>=0.8,<=0.8.99
alembic>=0.4.1
pyMySQL
basepython = python3.3
[testenv:py33-sa9]
[testenv:py33-sa9-pymysql]
deps = {[testenv]deps}
SQLAlchemy>=0.9,<=0.9.99
alembic>=0.4.1
basepython = python3.3
[testenv:py33-nosa]
deps = {[testenv]deps}
pyMySQL
basepython = python3.3