Retry on DBDeadLock affecting worker updates

Some updates on the workers table are prone to race conditions,
leading to DB deadlocks.

For example, when creating a large amount of volumes simultaneously,
some of them may hang in "Creating" state due to this issue.

We'll perform some retries when db deadlocks occur.

Change-Id: Iae211acd619070405f84a84ea58e88723a5e9f9b
Closes-Bug: #1789106
This commit is contained in:
Lucian Petrut 2018-10-19 15:02:17 +03:00
parent 7f39a12f7c
commit e1fc723db1
1 changed files with 4 additions and 0 deletions

View File

@ -1609,6 +1609,7 @@ def volume_attached(context, attachment_id, instance_uuid, host_name,
@handle_db_data_error
@require_context
@oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
def volume_create(context, values):
values['volume_metadata'] = _metadata_refs(values.get('metadata'),
models.VolumeMetadata)
@ -6928,6 +6929,7 @@ def _worker_set_updated_at_field(values):
values['updated_at'] = updated_at
@oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
def worker_create(context, **values):
"""Create a worker entry from optional arguments."""
_worker_set_updated_at_field(values)
@ -6964,6 +6966,7 @@ def _orm_worker_update(worker, values):
setattr(worker, key, value)
@oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
def worker_update(context, id, filters=None, orm_worker=None, **values):
"""Update a worker with given values."""
filters = filters or {}
@ -6982,6 +6985,7 @@ def worker_update(context, id, filters=None, orm_worker=None, **values):
return result
@oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
def worker_claim_for_cleanup(context, claimer_id, orm_worker):
"""Claim a worker entry for cleanup."""
# We set updated_at value so we are sure we update the DB entry even if the