openstack/nova
Code Issues Proposed changes

Raise InstanceMappingNotFound if StaleDataError is encountered

Browse Source 
We have a race where if a user issues a delete request while an
instance is in the middle of booting, we could fail to update the
'queued_for_delete' field on the instance mapping with:

  sqlalchemy.orm.exc.StaleDataError: UPDATE statement on table
  'instance_mappings' expected to update 1 row(s); 0 were matched.

This happens if we've retrieved the instance mapping record from the
database and then it gets deleted by nova-conductor before we attempt
to save() it.

This handles the situation by adding try-except around the update call
to catch StaleDataError and raise InstanceMappingNotFound instead,
which the caller does know how to handle.

Closes-Bug: #1882608

Change-Id: I2cdcad7226312ed81f4242c8d9ac919715524b48
(cherry picked from commit 16df22dcd5)
(cherry picked from commit 812ce632d5)
This commit is contained in:
melanie witt 2020-06-09 00:27:39 +00:00
parent 09d228d73c
commit 98048ee139
2 changed files with 20 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
nova/objects/instance_mapping.py
View File

@ -15,6 +15,7 @@ import collections
from oslo_log import log as logging
from oslo_utils import versionutils
import six
from sqlalchemy.orm import exc as orm_exc
from sqlalchemy.orm import joinedload
from sqlalchemy.sql import false
from sqlalchemy.sql import func
@ -161,8 +162,16 @@ class InstanceMapping(base.NovaTimestampObject, base.NovaObject):
def save(self):
changes = self.obj_get_changes()
changes = self._update_with_cell_id(changes)
db_mapping = self._save_in_db(self._context, self.instance_uuid,
changes)
try:
db_mapping = self._save_in_db(self._context, self.instance_uuid,
changes)
except orm_exc.StaleDataError:
# NOTE(melwitt): If the instance mapping has been deleted out from
# under us by conductor (delete requested while booting), we will
# encounter a StaleDataError after we retrieved the row and try to
# update it after it's been deleted. We can treat this like an
# instance mapping not found and allow the caller to handle it.
raise exception.InstanceMappingNotFound(uuid=self.instance_uuid)
self._from_db_object(self._context, self, db_mapping)
self.obj_reset_changes()

9
nova/tests/unit/objects/test_instance_mapping.py
View File

@ -12,6 +12,7 @@
import mock
from oslo_utils import uuidutils
from sqlalchemy.orm import exc as orm_exc
from nova import exception
from nova import objects
@ -151,6 +152,14 @@ class _TestInstanceMappingObject(object):
comparators={
'cell_mapping': self._check_cell_map_value})
@mock.patch.object(instance_mapping.InstanceMapping, '_save_in_db')
def test_save_stale_data_error(self, save_in_db):
save_in_db.side_effect = orm_exc.StaleDataError
mapping_obj = objects.InstanceMapping(self.context)
mapping_obj.instance_uuid = uuidutils.generate_uuid()
self.assertRaises(exception.InstanceMappingNotFound, mapping_obj.save)
@mock.patch.object(instance_mapping.InstanceMapping, '_destroy_in_db')
def test_destroy(self, destroy_in_db):
uuid = uuidutils.generate_uuid()