openstack
/
heat
Code Issues Proposed changes

Don't mark snapshot complete while task in progress 

Failures in the gate suggest that it is now somehow possible for the
Glance image created by a snapshot to appear to have been created even
while the Nova server remains in a state like 'image_uploading'. Servers
in this state cannot be rebuilt (to restore a snapshot). To ensure that
the snapshot has actually completed before the resource is marked
SNAPSHOT_COMPLETE, check the task state of the server as well as the
state of the snapshot image in Glance.

Change-Id: I2b5cddcbeab849054d84501dfcec48442f3666d3
Co-Authored-By: Rico Lin <rico.lin.guanyu@gmail.com>
Task: 36964
This commit is contained in:
Zane Bitter 2019-10-08 16:35:55 -04:00 committed by ricolin
parent 008346fd3a
commit 9dd523c311
2 changed files with 38 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
heat/engine/resources/openstack/nova/server.py
View File

@ -1748,7 +1748,11 @@ class Server(server_base.BaseServer, sh.SchedulerHintsMixin,
def check_snapshot_complete(self, image_id):
image = self.client_plugin('glance').get_image(image_id)
if image.status.lower() == self.IMAGE_ACTIVE:
return True
server = self.client_plugin().get_server(self.resource_id)
task_state = getattr(server, 'OS-EXT-STS:task_state', '')
if task_state not in {'image_uploading', 'image_snapshot_pending',
'image_snapshot', 'image_pending_upload'}:
return True
elif image.status.lower() in (self.IMAGE_ERROR, self.IMAGE_DELETED):
raise exception.Error(image.status)

36
heat/tests/openstack/nova/test_server.py
View File

@ -4191,16 +4191,46 @@ class ServersTest(common.HeatTestCase):
def test_server_check_snapshot_complete_fail(self):
self._test_server_check_snapshot_complete()
def test_server_check_snapshot_complete_with_not_complete_task_state(self):
for task_state in {'image_uploading', 'image_snapshot_pending',
'image_snapshot', 'image_pending_upload'}:
self._test_check_snapshot_complete_with_task_state(
task_state=task_state)
def test_server_check_snapshot_complete_with_active_task_state(self):
self._test_check_snapshot_complete_with_task_state()
def _test_check_snapshot_complete_with_task_state(self,
task_state='active'):
return_server = self.fc.servers.list()[1]
return_server.id = '1234'
server = self._create_test_server(return_server,
'test_server_snapshot')
image = mock.MagicMock(status='active')
self.patchobject(glance.GlanceClientPlugin, 'get_image',
return_value=image)
server_with_task_state = mock.Mock()
setattr(server_with_task_state, 'OS-EXT-STS:task_state', task_state)
mock_get = self.patchobject(
nova.NovaClientPlugin, 'get_server',
return_value=server_with_task_state)
if task_state not in {'image_uploading', 'image_snapshot_pending',
'image_snapshot', 'image_pending_upload'}:
self.assertTrue(server.check_snapshot_complete('fake_iamge_id'))
else:
self.assertFalse(server.check_snapshot_complete('fake_iamge_id'))
mock_get.assert_called_once_with(server.resource_id)
def _test_server_check_snapshot_complete(self, image_status='ERROR'):
return_server = self.fc.servers.list()[1]
return_server.id = '1234'
server = self._create_test_server(return_server,
'test_server_snapshot')
image_in_error = mock.Mock()
image_in_error.status = image_status
image_in_error = mock.MagicMock(status=image_status)
self.patchobject(glance.GlanceClientPlugin, 'get_image',
return_value=image_in_error)
self.assertRaises(exception.ResourceFailure,
scheduler.TaskRunner(server.snapshot))