Test that concurrent sqlalchemy transactions don't block

Neutron (and presumably other projects) expect to be able to have
overlapping transactions open from two eventlets at once, without
deadlock.

Note that the default mysql sqlalchemy engine (mysqldb) fails this test.

This change modifies py27 tox env to run the full test suite separately
with and without TEST_EVENTLET environment variable set.  When set,
tests are run with eventlet imported and monkey_patched.

Change-Id: Ib35c95defea8ace5b456af28801659f2ba67eb96
Related-Bug: #1350149
This commit is contained in:
Angus Lees 2014-07-02 18:23:19 +10:00
parent 9b79858453
commit 112aa3dd00
4 changed files with 159 additions and 0 deletions

View File

@ -0,0 +1,25 @@
# Copyright 2014 Rackspace
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
def should_run_eventlet_tests():
return bool(int(os.environ.get('TEST_EVENTLET') or '0'))
if should_run_eventlet_tests():
import eventlet
eventlet.monkey_patch()

View File

@ -0,0 +1,127 @@
# Copyright (c) 2014 Rackspace Hosting
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Unit tests for SQLAlchemy and eventlet interaction."""
import logging
import unittest2
from oslo_utils import importutils
import sqlalchemy as sa
from sqlalchemy.ext import declarative as sa_decl
from oslo_db import exception as db_exc
from oslo_db.sqlalchemy import models
from oslo_db.sqlalchemy import test_base
from oslo_db import tests
class EventletTestMixin(object):
def setUp(self):
super(EventletTestMixin, self).setUp()
BASE = sa_decl.declarative_base()
class TmpTable(BASE, models.ModelBase):
__tablename__ = 'test_async_eventlet'
id = sa.Column('id', sa.Integer, primary_key=True, nullable=False)
foo = sa.Column('foo', sa.Integer)
__table_args__ = (
sa.UniqueConstraint('foo', name='uniq_foo'),
)
self.test_table = TmpTable
TmpTable.__table__.create(self.engine)
self.addCleanup(lambda: TmpTable.__table__.drop(self.engine))
@unittest2.skipIf(not tests.should_run_eventlet_tests(),
'eventlet tests disabled unless TEST_EVENTLET=1')
def test_concurrent_transaction(self):
# Cause sqlalchemy to log executed SQL statements. Useful to
# determine exactly what and when was sent to DB.
sqla_logger = logging.getLogger('sqlalchemy.engine')
sqla_logger.setLevel(logging.INFO)
self.addCleanup(sqla_logger.setLevel, logging.NOTSET)
def operate_on_row(name, ready=None, proceed=None):
logging.debug('%s starting', name)
_session = self.sessionmaker()
with _session.begin():
logging.debug('%s ready', name)
# Modify the same row, inside transaction
tbl = self.test_table()
tbl.update({'foo': 10})
tbl.save(_session)
if ready is not None:
ready.send()
if proceed is not None:
logging.debug('%s waiting to proceed', name)
proceed.wait()
logging.debug('%s exiting transaction', name)
logging.debug('%s terminating', name)
return True
eventlet = importutils.try_import('eventlet')
if eventlet is None:
return self.skip('eventlet is required for this test')
a_ready = eventlet.event.Event()
a_proceed = eventlet.event.Event()
b_proceed = eventlet.event.Event()
# thread A opens transaction
logging.debug('spawning A')
a = eventlet.spawn(operate_on_row, 'A',
ready=a_ready, proceed=a_proceed)
logging.debug('waiting for A to enter transaction')
a_ready.wait()
# thread B opens transaction on same row
logging.debug('spawning B')
b = eventlet.spawn(operate_on_row, 'B',
proceed=b_proceed)
logging.debug('waiting for B to (attempt to) enter transaction')
eventlet.sleep(1) # should(?) advance B to blocking on transaction
# While B is still blocked, A should be able to proceed
a_proceed.send()
# Will block forever(*) if DB library isn't reentrant.
# (*) Until some form of timeout/deadlock detection kicks in.
# This is the key test that async is working. If this hangs
# (or raises a timeout/deadlock exception), then you have failed
# this test.
self.assertTrue(a.wait())
b_proceed.send()
# If everything proceeded without blocking, B will throw a
# "duplicate entry" exception when it tries to insert the same row
self.assertRaises(db_exc.DBDuplicateEntry, b.wait)
# Note that sqlite fails the above concurrency tests, and is not
# mentioned below.
# ie: This file performs no tests by default.
class MySQLEventletTestCase(EventletTestMixin,
test_base.MySQLOpportunisticTestCase):
pass
class PostgreSQLEventletTestCase(EventletTestMixin,
test_base.PostgreSQLOpportunisticTestCase):
pass

View File

@ -7,6 +7,7 @@ hacking<0.11,>=0.10.0
coverage>=3.6 coverage>=3.6
discover discover
doc8 # Apache-2.0 doc8 # Apache-2.0
eventlet>=0.17.4
fixtures>=1.3.1 fixtures>=1.3.1
PyMySQL>=0.6.2 # MIT License PyMySQL>=0.6.2 # MIT License
psycopg2 psycopg2

View File

@ -10,6 +10,7 @@ envlist = py26,py27,py34,pep8,pip-missing-reqs
# for oslo libraries because of the namespace package. # for oslo libraries because of the namespace package.
#usedevelop = True #usedevelop = True
whitelist_externals = bash whitelist_externals = bash
env
install_command = pip install -U {opts} {packages} install_command = pip install -U {opts} {packages}
setenv = setenv =
VIRTUAL_ENV={envdir} VIRTUAL_ENV={envdir}
@ -21,6 +22,11 @@ commands = bash tools/pretty_tox.sh '{posargs}'
commands = pip install SQLAlchemy>=0.9.0,!=0.9.5,<1.0.0 commands = pip install SQLAlchemy>=0.9.0,!=0.9.5,<1.0.0
python setup.py testr --slowest --testr-args='{posargs}' python setup.py testr --slowest --testr-args='{posargs}'
[testenv:py27]
commands =
env TEST_EVENTLET=0 bash tools/pretty_tox.sh '{posargs}'
env TEST_EVENTLET=1 bash tools/pretty_tox.sh '{posargs}'
[testenv:mysql-python] [testenv:mysql-python]
setenv = setenv =
{[testenv]setenv} {[testenv]setenv}