Check for StaleData errors in retry decorator

If an object is updated in a compare and swap fashion, it
can fail when the change is flushed and raise a
StaleDataError if another process has updated the SQL record
at the same time. We need to catch these with the retry
decorator to restart the update process in a new transaction
with a fresh read of the latest state.

Partially-Implements: bp/push-notifications
Change-Id: I151ffcf47926f5ac66e452974f87e8bc2a906151
This commit is contained in:
Kevin Benton 2016-06-07 14:35:47 -07:00
parent b4e99a2e12
commit 948461c8b2
4 changed files with 15 additions and 5 deletions

View File

@ -15,6 +15,7 @@
import contextlib
from debtcollector import moves
from oslo_config import cfg
from oslo_db import api as oslo_db_api
from oslo_db import exception as db_exc
@ -22,6 +23,7 @@ from oslo_db.sqlalchemy import enginefacade
from oslo_utils import excutils
import osprofiler.sqlalchemy
import sqlalchemy
from sqlalchemy.orm import exc
from neutron.common import exceptions
from neutron.common import profiler # noqa
@ -34,13 +36,16 @@ _FACADE = None
MAX_RETRIES = 10
def is_deadlock(exc):
return _is_nested_instance(exc, db_exc.DBDeadlock)
def is_retriable(e):
return _is_nested_instance(e, (db_exc.DBDeadlock, exc.StaleDataError))
is_deadlock = moves.moved_function(is_retriable, 'is_deadlock', __name__,
message='use "is_retriable" instead',
version='newton', removal_version='ocata')
retry_db_errors = oslo_db_api.wrap_db_retry(
max_retries=MAX_RETRIES,
retry_on_request=True,
exception_checker=is_deadlock
exception_checker=is_retriable
)

View File

@ -161,7 +161,7 @@ class DbQuotaDriver(object):
retry_interval=0.1,
inc_retry_interval=True,
retry_on_request=True,
exception_checker=db_api.is_deadlock)
exception_checker=db_api.is_retriable)
def make_reservation(self, context, tenant_id, resources, deltas, plugin):
# Lock current reservation table
# NOTE(salv-orlando): This routine uses DB write locks.

View File

@ -139,7 +139,7 @@ class TunnelTypeDriver(helpers.SegmentTypeDriver):
@oslo_db_api.wrap_db_retry(
max_retries=db_api.MAX_RETRIES,
exception_checker=db_api.is_deadlock)
exception_checker=db_api.is_retriable)
def sync_allocations(self):
# determine current configured allocatable tunnel ids
tunnel_ids = set()

View File

@ -13,6 +13,7 @@
# limitations under the License.
from oslo_db import exception as db_exc
from sqlalchemy.orm import exc
import testtools
from neutron.common import exceptions
@ -64,6 +65,10 @@ class TestDeadLockDecorator(base.BaseTestCase):
with testtools.ExpectedException(ValueError):
self._decorated_function(1, ValueError)
def test_staledata_error_caught(self):
e = exc.StaleDataError()
self.assertIsNone(self._decorated_function(1, e))
def test_multi_exception_contains_deadlock(self):
e = exceptions.MultipleExceptions([ValueError(), db_exc.DBDeadlock()])
self.assertIsNone(self._decorated_function(1, e))