From 812ce632d50bfc32de62d544746e0b9a83d90ab7 Mon Sep 17 00:00:00 2001 From: melanie witt Date: Tue, 9 Jun 2020 00:27:39 +0000 Subject: [PATCH] Raise InstanceMappingNotFound if StaleDataError is encountered We have a race where if a user issues a delete request while an instance is in the middle of booting, we could fail to update the 'queued_for_delete' field on the instance mapping with: sqlalchemy.orm.exc.StaleDataError: UPDATE statement on table 'instance_mappings' expected to update 1 row(s); 0 were matched. This happens if we've retrieved the instance mapping record from the database and then it gets deleted by nova-conductor before we attempt to save() it. This handles the situation by adding try-except around the update call to catch StaleDataError and raise InstanceMappingNotFound instead, which the caller does know how to handle. Closes-Bug: #1882608 Change-Id: I2cdcad7226312ed81f4242c8d9ac919715524b48 (cherry picked from commit 16df22dcd57a73fe3be15c64c41b4081b4826ef2) --- nova/objects/instance_mapping.py | 13 +++++++++++-- nova/tests/unit/objects/test_instance_mapping.py | 9 +++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/nova/objects/instance_mapping.py b/nova/objects/instance_mapping.py index 0392f04770a0..8b5f6ba92e70 100644 --- a/nova/objects/instance_mapping.py +++ b/nova/objects/instance_mapping.py @@ -15,6 +15,7 @@ import collections from oslo_log import log as logging from oslo_utils import versionutils import six +from sqlalchemy.orm import exc as orm_exc from sqlalchemy.orm import joinedload from sqlalchemy.sql import false from sqlalchemy.sql import func @@ -161,8 +162,16 @@ class InstanceMapping(base.NovaTimestampObject, base.NovaObject): def save(self): changes = self.obj_get_changes() changes = self._update_with_cell_id(changes) - db_mapping = self._save_in_db(self._context, self.instance_uuid, - changes) + try: + db_mapping = self._save_in_db(self._context, self.instance_uuid, + changes) + except orm_exc.StaleDataError: + # NOTE(melwitt): If the instance mapping has been deleted out from + # under us by conductor (delete requested while booting), we will + # encounter a StaleDataError after we retrieved the row and try to + # update it after it's been deleted. We can treat this like an + # instance mapping not found and allow the caller to handle it. + raise exception.InstanceMappingNotFound(uuid=self.instance_uuid) self._from_db_object(self._context, self, db_mapping) self.obj_reset_changes() diff --git a/nova/tests/unit/objects/test_instance_mapping.py b/nova/tests/unit/objects/test_instance_mapping.py index ec50517a20c5..2c877c0a1f55 100644 --- a/nova/tests/unit/objects/test_instance_mapping.py +++ b/nova/tests/unit/objects/test_instance_mapping.py @@ -12,6 +12,7 @@ import mock from oslo_utils import uuidutils +from sqlalchemy.orm import exc as orm_exc from nova import exception from nova import objects @@ -151,6 +152,14 @@ class _TestInstanceMappingObject(object): comparators={ 'cell_mapping': self._check_cell_map_value}) + @mock.patch.object(instance_mapping.InstanceMapping, '_save_in_db') + def test_save_stale_data_error(self, save_in_db): + save_in_db.side_effect = orm_exc.StaleDataError + mapping_obj = objects.InstanceMapping(self.context) + mapping_obj.instance_uuid = uuidutils.generate_uuid() + + self.assertRaises(exception.InstanceMappingNotFound, mapping_obj.save) + @mock.patch.object(instance_mapping.InstanceMapping, '_destroy_in_db') def test_destroy(self, destroy_in_db): uuid = uuidutils.generate_uuid()