openstack
/
magnum
Code Issues Proposed changes

Failed state was ignored for default ngs 

This change addresses an issue with state aggregation where default
ngs were in a failed state and it was ignored. e.g. default ngs were
in UPDATE_FAILED, a non-default ng in UPDATE_COMPLETE and the cluster
reported UPDATE_COMPLETE.

Change-Id: I317c896f0f161427fada677393df5fd2435e7bbd
story: 2006713
task: 37084
This commit is contained in:
Theodoros Tsioutsias 2019-10-14 07:48:46 +00:00
parent 7dc4c7d904
commit ae159882e4
2 changed files with 114 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
magnum/drivers/heat/driver.py
View File

@ -411,30 +411,28 @@ class HeatPoller(object):
IN_PROGRESS = '_IN_PROGRESS'
COMPLETE = '_COMPLETE'
UPDATE = 'UPDATE'
DELETE = 'DELETE'
previous_state = self.cluster.status
self.cluster.status_reason = None
non_default_ngs_exist = any(not ns.is_default for ns in ng_statuses)
# Both default nodegroups will have the same status so it's
# enough to check one of them.
self.cluster.status = self.cluster.default_ng_master.status
default_ng = self.cluster.default_ng_master
if (default_ng.status.endswith(IN_PROGRESS) or
default_ng.status == fields.ClusterStatus.DELETE_COMPLETE):
self.cluster.save()
return
default_ng_status = self.cluster.default_ng_master.status
# Whatever action is going on in a cluster that has
# non-default ngs, we call it update except for delete.
action = DELETE if default_ng_status.startswith(DELETE) else UPDATE
# Keep priority to the states below
for state in (IN_PROGRESS, FAILED, COMPLETE):
if any(ns.status.endswith(state) for ns in ng_statuses
if not ns.is_default):
status = getattr(fields.ClusterStatus, UPDATE+state)
if any(ns.status.endswith(state) for ns in ng_statuses):
if non_default_ngs_exist:
status = getattr(fields.ClusterStatus, action+state)
else:
# If there are no non-default NGs
# just use the default NG's status.
status = default_ng_status
self.cluster.status = status
if state == FAILED:
reasons = ["%s failed" % (ns.name)
for ns in ng_statuses
if ns.status.endswith(FAILED)]
self.cluster.status_reason = ' ,'.join(reasons)
break
if self.cluster.status == fields.ClusterStatus.CREATE_COMPLETE:
@ -449,6 +447,13 @@ class HeatPoller(object):
fields.ClusterStatus.CREATE_IN_PROGRESS):
self.cluster.status = fields.ClusterStatus.UPDATE_COMPLETE
# Summarize the failed reasons.
if self.cluster.status.endswith(FAILED):
reasons = ["%s failed" % (ns.name)
for ns in ng_statuses
if ns.status.endswith(FAILED)]
self.cluster.status_reason = ' ,'.join(reasons)
self.cluster.save()
def _delete_complete(self):

97
magnum/tests/unit/drivers/test_heat_driver.py
View File

@ -33,7 +33,7 @@ class TestHeatPoller(base.TestCase):
self.mock_stacks = dict()
self.def_ngs = list()
def _create_nodegroup(self, cluster, uuid, stack_id, role=None,
def _create_nodegroup(self, cluster, uuid, stack_id, name=None, role=None,
is_default=False, stack_status=None,
status_reason=None, stack_params=None,
stack_missing=False):
@ -45,6 +45,8 @@ class TestHeatPoller(base.TestCase):
role = 'worker' if role is None else role
ng = mock.MagicMock(uuid=uuid, role=role, is_default=is_default,
stack_id=stack_id)
if name is not None:
type(ng).name = name
cluster.nodegroups.append(ng)
@ -88,13 +90,15 @@ class TestHeatPoller(base.TestCase):
cluster = mock.MagicMock(nodegroups=list())
def_worker = self._create_nodegroup(cluster, 'worker_ng', 'stack1',
role='worker', is_default=True,
name='worker_ng', role='worker',
is_default=True,
stack_status=default_stack_status,
status_reason=status_reason,
stack_params=stack_params,
stack_missing=stack_missing)
def_master = self._create_nodegroup(cluster, 'master_ng', 'stack1',
role='master', is_default=True,
name='master_ng', role='master',
is_default=True,
stack_status=default_stack_status,
status_reason=status_reason,
stack_params=stack_params,
@ -669,3 +673,90 @@ class TestHeatPoller(base.TestCase):
for def_ng in self.def_ngs:
self.assertEqual(cluster_status.CREATE_COMPLETE, def_ng.status)
self.assertEqual(cluster_status.DELETE_COMPLETE, ng.status)
def test_poll_and_check_failed_default_ng(self):
cluster, poller = self.setup_poll_test(
default_stack_status=cluster_status.UPDATE_FAILED)
ng = self._create_nodegroup(
cluster, 'ng', 'stack2',
stack_status=cluster_status.UPDATE_COMPLETE)
cluster.status = cluster_status.UPDATE_IN_PROGRESS
poller.poll_and_check()
for def_ng in self.def_ngs:
self.assertEqual(cluster_status.UPDATE_FAILED, def_ng.status)
self.assertEqual(2, def_ng.save.call_count)
self.assertEqual(cluster_status.UPDATE_COMPLETE, ng.status)
self.assertEqual(1, ng.save.call_count)
self.assertEqual(cluster_status.UPDATE_FAILED, cluster.status)
self.assertEqual(1, cluster.save.call_count)
def test_poll_and_check_rollback_failed_default_ng(self):
cluster, poller = self.setup_poll_test(
default_stack_status=cluster_status.ROLLBACK_FAILED)
ng = self._create_nodegroup(
cluster, 'ng', 'stack2',
stack_status=cluster_status.UPDATE_COMPLETE)
cluster.status = cluster_status.UPDATE_IN_PROGRESS
poller.poll_and_check()
for def_ng in self.def_ngs:
self.assertEqual(cluster_status.ROLLBACK_FAILED, def_ng.status)
self.assertEqual(2, def_ng.save.call_count)
self.assertEqual(cluster_status.UPDATE_COMPLETE, ng.status)
self.assertEqual(1, ng.save.call_count)
self.assertEqual(cluster_status.UPDATE_FAILED, cluster.status)
self.assertEqual(1, cluster.save.call_count)
def test_poll_and_check_rollback_failed_def_ng(self):
cluster, poller = self.setup_poll_test(
default_stack_status=cluster_status.DELETE_FAILED)
ng = self._create_nodegroup(
cluster, 'ng', 'stack2',
stack_status=cluster_status.DELETE_IN_PROGRESS)
cluster.status = cluster_status.DELETE_IN_PROGRESS
poller.poll_and_check()
for def_ng in self.def_ngs:
self.assertEqual(cluster_status.DELETE_FAILED, def_ng.status)
self.assertEqual(2, def_ng.save.call_count)
self.assertEqual(cluster_status.DELETE_IN_PROGRESS, ng.status)
self.assertEqual(1, ng.save.call_count)
self.assertEqual(cluster_status.DELETE_IN_PROGRESS, cluster.status)
self.assertEqual(1, cluster.save.call_count)
def test_poll_and_check_delete_failed_def_ng(self):
cluster, poller = self.setup_poll_test(
default_stack_status=cluster_status.DELETE_FAILED)
ng = self._create_nodegroup(
cluster, 'ng', 'stack2',
stack_status=cluster_status.DELETE_COMPLETE)
cluster.status = cluster_status.DELETE_IN_PROGRESS
poller.poll_and_check()
for def_ng in self.def_ngs:
self.assertEqual(cluster_status.DELETE_FAILED, def_ng.status)
self.assertEqual(2, def_ng.save.call_count)
# Check that the non-default ng was deleted
self.assertEqual(1, ng.destroy.call_count)
self.assertEqual(cluster_status.DELETE_FAILED, cluster.status)
self.assertEqual(1, cluster.save.call_count)
self.assertIn('worker_ng', cluster.status_reason)
self.assertIn('master_ng', cluster.status_reason)