Failed state was ignored for default ngs
This change addresses an issue with state aggregation where default ngs were in a failed state and it was ignored. e.g. default ngs were in UPDATE_FAILED, a non-default ng in UPDATE_COMPLETE and the cluster reported UPDATE_COMPLETE. Change-Id: I317c896f0f161427fada677393df5fd2435e7bbd story: 2006713 task: 37084
This commit is contained in:
parent
7dc4c7d904
commit
ae159882e4
@ -411,30 +411,28 @@ class HeatPoller(object):
|
|||||||
IN_PROGRESS = '_IN_PROGRESS'
|
IN_PROGRESS = '_IN_PROGRESS'
|
||||||
COMPLETE = '_COMPLETE'
|
COMPLETE = '_COMPLETE'
|
||||||
UPDATE = 'UPDATE'
|
UPDATE = 'UPDATE'
|
||||||
|
DELETE = 'DELETE'
|
||||||
|
|
||||||
previous_state = self.cluster.status
|
previous_state = self.cluster.status
|
||||||
self.cluster.status_reason = None
|
self.cluster.status_reason = None
|
||||||
|
|
||||||
|
non_default_ngs_exist = any(not ns.is_default for ns in ng_statuses)
|
||||||
# Both default nodegroups will have the same status so it's
|
# Both default nodegroups will have the same status so it's
|
||||||
# enough to check one of them.
|
# enough to check one of them.
|
||||||
self.cluster.status = self.cluster.default_ng_master.status
|
default_ng_status = self.cluster.default_ng_master.status
|
||||||
default_ng = self.cluster.default_ng_master
|
# Whatever action is going on in a cluster that has
|
||||||
if (default_ng.status.endswith(IN_PROGRESS) or
|
# non-default ngs, we call it update except for delete.
|
||||||
default_ng.status == fields.ClusterStatus.DELETE_COMPLETE):
|
action = DELETE if default_ng_status.startswith(DELETE) else UPDATE
|
||||||
self.cluster.save()
|
|
||||||
return
|
|
||||||
|
|
||||||
# Keep priority to the states below
|
# Keep priority to the states below
|
||||||
for state in (IN_PROGRESS, FAILED, COMPLETE):
|
for state in (IN_PROGRESS, FAILED, COMPLETE):
|
||||||
if any(ns.status.endswith(state) for ns in ng_statuses
|
if any(ns.status.endswith(state) for ns in ng_statuses):
|
||||||
if not ns.is_default):
|
if non_default_ngs_exist:
|
||||||
status = getattr(fields.ClusterStatus, UPDATE+state)
|
status = getattr(fields.ClusterStatus, action+state)
|
||||||
|
else:
|
||||||
|
# If there are no non-default NGs
|
||||||
|
# just use the default NG's status.
|
||||||
|
status = default_ng_status
|
||||||
self.cluster.status = status
|
self.cluster.status = status
|
||||||
if state == FAILED:
|
|
||||||
reasons = ["%s failed" % (ns.name)
|
|
||||||
for ns in ng_statuses
|
|
||||||
if ns.status.endswith(FAILED)]
|
|
||||||
self.cluster.status_reason = ' ,'.join(reasons)
|
|
||||||
break
|
break
|
||||||
|
|
||||||
if self.cluster.status == fields.ClusterStatus.CREATE_COMPLETE:
|
if self.cluster.status == fields.ClusterStatus.CREATE_COMPLETE:
|
||||||
@ -449,6 +447,13 @@ class HeatPoller(object):
|
|||||||
fields.ClusterStatus.CREATE_IN_PROGRESS):
|
fields.ClusterStatus.CREATE_IN_PROGRESS):
|
||||||
self.cluster.status = fields.ClusterStatus.UPDATE_COMPLETE
|
self.cluster.status = fields.ClusterStatus.UPDATE_COMPLETE
|
||||||
|
|
||||||
|
# Summarize the failed reasons.
|
||||||
|
if self.cluster.status.endswith(FAILED):
|
||||||
|
reasons = ["%s failed" % (ns.name)
|
||||||
|
for ns in ng_statuses
|
||||||
|
if ns.status.endswith(FAILED)]
|
||||||
|
self.cluster.status_reason = ' ,'.join(reasons)
|
||||||
|
|
||||||
self.cluster.save()
|
self.cluster.save()
|
||||||
|
|
||||||
def _delete_complete(self):
|
def _delete_complete(self):
|
||||||
|
@ -33,7 +33,7 @@ class TestHeatPoller(base.TestCase):
|
|||||||
self.mock_stacks = dict()
|
self.mock_stacks = dict()
|
||||||
self.def_ngs = list()
|
self.def_ngs = list()
|
||||||
|
|
||||||
def _create_nodegroup(self, cluster, uuid, stack_id, role=None,
|
def _create_nodegroup(self, cluster, uuid, stack_id, name=None, role=None,
|
||||||
is_default=False, stack_status=None,
|
is_default=False, stack_status=None,
|
||||||
status_reason=None, stack_params=None,
|
status_reason=None, stack_params=None,
|
||||||
stack_missing=False):
|
stack_missing=False):
|
||||||
@ -45,6 +45,8 @@ class TestHeatPoller(base.TestCase):
|
|||||||
role = 'worker' if role is None else role
|
role = 'worker' if role is None else role
|
||||||
ng = mock.MagicMock(uuid=uuid, role=role, is_default=is_default,
|
ng = mock.MagicMock(uuid=uuid, role=role, is_default=is_default,
|
||||||
stack_id=stack_id)
|
stack_id=stack_id)
|
||||||
|
if name is not None:
|
||||||
|
type(ng).name = name
|
||||||
|
|
||||||
cluster.nodegroups.append(ng)
|
cluster.nodegroups.append(ng)
|
||||||
|
|
||||||
@ -88,13 +90,15 @@ class TestHeatPoller(base.TestCase):
|
|||||||
cluster = mock.MagicMock(nodegroups=list())
|
cluster = mock.MagicMock(nodegroups=list())
|
||||||
|
|
||||||
def_worker = self._create_nodegroup(cluster, 'worker_ng', 'stack1',
|
def_worker = self._create_nodegroup(cluster, 'worker_ng', 'stack1',
|
||||||
role='worker', is_default=True,
|
name='worker_ng', role='worker',
|
||||||
|
is_default=True,
|
||||||
stack_status=default_stack_status,
|
stack_status=default_stack_status,
|
||||||
status_reason=status_reason,
|
status_reason=status_reason,
|
||||||
stack_params=stack_params,
|
stack_params=stack_params,
|
||||||
stack_missing=stack_missing)
|
stack_missing=stack_missing)
|
||||||
def_master = self._create_nodegroup(cluster, 'master_ng', 'stack1',
|
def_master = self._create_nodegroup(cluster, 'master_ng', 'stack1',
|
||||||
role='master', is_default=True,
|
name='master_ng', role='master',
|
||||||
|
is_default=True,
|
||||||
stack_status=default_stack_status,
|
stack_status=default_stack_status,
|
||||||
status_reason=status_reason,
|
status_reason=status_reason,
|
||||||
stack_params=stack_params,
|
stack_params=stack_params,
|
||||||
@ -669,3 +673,90 @@ class TestHeatPoller(base.TestCase):
|
|||||||
for def_ng in self.def_ngs:
|
for def_ng in self.def_ngs:
|
||||||
self.assertEqual(cluster_status.CREATE_COMPLETE, def_ng.status)
|
self.assertEqual(cluster_status.CREATE_COMPLETE, def_ng.status)
|
||||||
self.assertEqual(cluster_status.DELETE_COMPLETE, ng.status)
|
self.assertEqual(cluster_status.DELETE_COMPLETE, ng.status)
|
||||||
|
|
||||||
|
def test_poll_and_check_failed_default_ng(self):
|
||||||
|
cluster, poller = self.setup_poll_test(
|
||||||
|
default_stack_status=cluster_status.UPDATE_FAILED)
|
||||||
|
|
||||||
|
ng = self._create_nodegroup(
|
||||||
|
cluster, 'ng', 'stack2',
|
||||||
|
stack_status=cluster_status.UPDATE_COMPLETE)
|
||||||
|
|
||||||
|
cluster.status = cluster_status.UPDATE_IN_PROGRESS
|
||||||
|
poller.poll_and_check()
|
||||||
|
|
||||||
|
for def_ng in self.def_ngs:
|
||||||
|
self.assertEqual(cluster_status.UPDATE_FAILED, def_ng.status)
|
||||||
|
self.assertEqual(2, def_ng.save.call_count)
|
||||||
|
|
||||||
|
self.assertEqual(cluster_status.UPDATE_COMPLETE, ng.status)
|
||||||
|
self.assertEqual(1, ng.save.call_count)
|
||||||
|
|
||||||
|
self.assertEqual(cluster_status.UPDATE_FAILED, cluster.status)
|
||||||
|
self.assertEqual(1, cluster.save.call_count)
|
||||||
|
|
||||||
|
def test_poll_and_check_rollback_failed_default_ng(self):
|
||||||
|
cluster, poller = self.setup_poll_test(
|
||||||
|
default_stack_status=cluster_status.ROLLBACK_FAILED)
|
||||||
|
|
||||||
|
ng = self._create_nodegroup(
|
||||||
|
cluster, 'ng', 'stack2',
|
||||||
|
stack_status=cluster_status.UPDATE_COMPLETE)
|
||||||
|
|
||||||
|
cluster.status = cluster_status.UPDATE_IN_PROGRESS
|
||||||
|
poller.poll_and_check()
|
||||||
|
|
||||||
|
for def_ng in self.def_ngs:
|
||||||
|
self.assertEqual(cluster_status.ROLLBACK_FAILED, def_ng.status)
|
||||||
|
self.assertEqual(2, def_ng.save.call_count)
|
||||||
|
|
||||||
|
self.assertEqual(cluster_status.UPDATE_COMPLETE, ng.status)
|
||||||
|
self.assertEqual(1, ng.save.call_count)
|
||||||
|
|
||||||
|
self.assertEqual(cluster_status.UPDATE_FAILED, cluster.status)
|
||||||
|
self.assertEqual(1, cluster.save.call_count)
|
||||||
|
|
||||||
|
def test_poll_and_check_rollback_failed_def_ng(self):
|
||||||
|
cluster, poller = self.setup_poll_test(
|
||||||
|
default_stack_status=cluster_status.DELETE_FAILED)
|
||||||
|
|
||||||
|
ng = self._create_nodegroup(
|
||||||
|
cluster, 'ng', 'stack2',
|
||||||
|
stack_status=cluster_status.DELETE_IN_PROGRESS)
|
||||||
|
|
||||||
|
cluster.status = cluster_status.DELETE_IN_PROGRESS
|
||||||
|
poller.poll_and_check()
|
||||||
|
|
||||||
|
for def_ng in self.def_ngs:
|
||||||
|
self.assertEqual(cluster_status.DELETE_FAILED, def_ng.status)
|
||||||
|
self.assertEqual(2, def_ng.save.call_count)
|
||||||
|
|
||||||
|
self.assertEqual(cluster_status.DELETE_IN_PROGRESS, ng.status)
|
||||||
|
self.assertEqual(1, ng.save.call_count)
|
||||||
|
|
||||||
|
self.assertEqual(cluster_status.DELETE_IN_PROGRESS, cluster.status)
|
||||||
|
self.assertEqual(1, cluster.save.call_count)
|
||||||
|
|
||||||
|
def test_poll_and_check_delete_failed_def_ng(self):
|
||||||
|
cluster, poller = self.setup_poll_test(
|
||||||
|
default_stack_status=cluster_status.DELETE_FAILED)
|
||||||
|
|
||||||
|
ng = self._create_nodegroup(
|
||||||
|
cluster, 'ng', 'stack2',
|
||||||
|
stack_status=cluster_status.DELETE_COMPLETE)
|
||||||
|
|
||||||
|
cluster.status = cluster_status.DELETE_IN_PROGRESS
|
||||||
|
poller.poll_and_check()
|
||||||
|
|
||||||
|
for def_ng in self.def_ngs:
|
||||||
|
self.assertEqual(cluster_status.DELETE_FAILED, def_ng.status)
|
||||||
|
self.assertEqual(2, def_ng.save.call_count)
|
||||||
|
|
||||||
|
# Check that the non-default ng was deleted
|
||||||
|
self.assertEqual(1, ng.destroy.call_count)
|
||||||
|
|
||||||
|
self.assertEqual(cluster_status.DELETE_FAILED, cluster.status)
|
||||||
|
self.assertEqual(1, cluster.save.call_count)
|
||||||
|
|
||||||
|
self.assertIn('worker_ng', cluster.status_reason)
|
||||||
|
self.assertIn('master_ng', cluster.status_reason)
|
||||||
|
Loading…
Reference in New Issue
Block a user