From ae159882e45f124ae0c3431f6f6a0b3d6713222b Mon Sep 17 00:00:00 2001 From: Theodoros Tsioutsias Date: Mon, 14 Oct 2019 07:48:46 +0000 Subject: [PATCH] Failed state was ignored for default ngs This change addresses an issue with state aggregation where default ngs were in a failed state and it was ignored. e.g. default ngs were in UPDATE_FAILED, a non-default ng in UPDATE_COMPLETE and the cluster reported UPDATE_COMPLETE. Change-Id: I317c896f0f161427fada677393df5fd2435e7bbd story: 2006713 task: 37084 --- magnum/drivers/heat/driver.py | 35 ++++--- magnum/tests/unit/drivers/test_heat_driver.py | 97 ++++++++++++++++++- 2 files changed, 114 insertions(+), 18 deletions(-) diff --git a/magnum/drivers/heat/driver.py b/magnum/drivers/heat/driver.py index 33a243af01..431d5de547 100755 --- a/magnum/drivers/heat/driver.py +++ b/magnum/drivers/heat/driver.py @@ -411,30 +411,28 @@ class HeatPoller(object): IN_PROGRESS = '_IN_PROGRESS' COMPLETE = '_COMPLETE' UPDATE = 'UPDATE' + DELETE = 'DELETE' previous_state = self.cluster.status self.cluster.status_reason = None + non_default_ngs_exist = any(not ns.is_default for ns in ng_statuses) # Both default nodegroups will have the same status so it's # enough to check one of them. - self.cluster.status = self.cluster.default_ng_master.status - default_ng = self.cluster.default_ng_master - if (default_ng.status.endswith(IN_PROGRESS) or - default_ng.status == fields.ClusterStatus.DELETE_COMPLETE): - self.cluster.save() - return - + default_ng_status = self.cluster.default_ng_master.status + # Whatever action is going on in a cluster that has + # non-default ngs, we call it update except for delete. + action = DELETE if default_ng_status.startswith(DELETE) else UPDATE # Keep priority to the states below for state in (IN_PROGRESS, FAILED, COMPLETE): - if any(ns.status.endswith(state) for ns in ng_statuses - if not ns.is_default): - status = getattr(fields.ClusterStatus, UPDATE+state) + if any(ns.status.endswith(state) for ns in ng_statuses): + if non_default_ngs_exist: + status = getattr(fields.ClusterStatus, action+state) + else: + # If there are no non-default NGs + # just use the default NG's status. + status = default_ng_status self.cluster.status = status - if state == FAILED: - reasons = ["%s failed" % (ns.name) - for ns in ng_statuses - if ns.status.endswith(FAILED)] - self.cluster.status_reason = ' ,'.join(reasons) break if self.cluster.status == fields.ClusterStatus.CREATE_COMPLETE: @@ -449,6 +447,13 @@ class HeatPoller(object): fields.ClusterStatus.CREATE_IN_PROGRESS): self.cluster.status = fields.ClusterStatus.UPDATE_COMPLETE + # Summarize the failed reasons. + if self.cluster.status.endswith(FAILED): + reasons = ["%s failed" % (ns.name) + for ns in ng_statuses + if ns.status.endswith(FAILED)] + self.cluster.status_reason = ' ,'.join(reasons) + self.cluster.save() def _delete_complete(self): diff --git a/magnum/tests/unit/drivers/test_heat_driver.py b/magnum/tests/unit/drivers/test_heat_driver.py index e136750a87..04d46fd231 100644 --- a/magnum/tests/unit/drivers/test_heat_driver.py +++ b/magnum/tests/unit/drivers/test_heat_driver.py @@ -33,7 +33,7 @@ class TestHeatPoller(base.TestCase): self.mock_stacks = dict() self.def_ngs = list() - def _create_nodegroup(self, cluster, uuid, stack_id, role=None, + def _create_nodegroup(self, cluster, uuid, stack_id, name=None, role=None, is_default=False, stack_status=None, status_reason=None, stack_params=None, stack_missing=False): @@ -45,6 +45,8 @@ class TestHeatPoller(base.TestCase): role = 'worker' if role is None else role ng = mock.MagicMock(uuid=uuid, role=role, is_default=is_default, stack_id=stack_id) + if name is not None: + type(ng).name = name cluster.nodegroups.append(ng) @@ -88,13 +90,15 @@ class TestHeatPoller(base.TestCase): cluster = mock.MagicMock(nodegroups=list()) def_worker = self._create_nodegroup(cluster, 'worker_ng', 'stack1', - role='worker', is_default=True, + name='worker_ng', role='worker', + is_default=True, stack_status=default_stack_status, status_reason=status_reason, stack_params=stack_params, stack_missing=stack_missing) def_master = self._create_nodegroup(cluster, 'master_ng', 'stack1', - role='master', is_default=True, + name='master_ng', role='master', + is_default=True, stack_status=default_stack_status, status_reason=status_reason, stack_params=stack_params, @@ -669,3 +673,90 @@ class TestHeatPoller(base.TestCase): for def_ng in self.def_ngs: self.assertEqual(cluster_status.CREATE_COMPLETE, def_ng.status) self.assertEqual(cluster_status.DELETE_COMPLETE, ng.status) + + def test_poll_and_check_failed_default_ng(self): + cluster, poller = self.setup_poll_test( + default_stack_status=cluster_status.UPDATE_FAILED) + + ng = self._create_nodegroup( + cluster, 'ng', 'stack2', + stack_status=cluster_status.UPDATE_COMPLETE) + + cluster.status = cluster_status.UPDATE_IN_PROGRESS + poller.poll_and_check() + + for def_ng in self.def_ngs: + self.assertEqual(cluster_status.UPDATE_FAILED, def_ng.status) + self.assertEqual(2, def_ng.save.call_count) + + self.assertEqual(cluster_status.UPDATE_COMPLETE, ng.status) + self.assertEqual(1, ng.save.call_count) + + self.assertEqual(cluster_status.UPDATE_FAILED, cluster.status) + self.assertEqual(1, cluster.save.call_count) + + def test_poll_and_check_rollback_failed_default_ng(self): + cluster, poller = self.setup_poll_test( + default_stack_status=cluster_status.ROLLBACK_FAILED) + + ng = self._create_nodegroup( + cluster, 'ng', 'stack2', + stack_status=cluster_status.UPDATE_COMPLETE) + + cluster.status = cluster_status.UPDATE_IN_PROGRESS + poller.poll_and_check() + + for def_ng in self.def_ngs: + self.assertEqual(cluster_status.ROLLBACK_FAILED, def_ng.status) + self.assertEqual(2, def_ng.save.call_count) + + self.assertEqual(cluster_status.UPDATE_COMPLETE, ng.status) + self.assertEqual(1, ng.save.call_count) + + self.assertEqual(cluster_status.UPDATE_FAILED, cluster.status) + self.assertEqual(1, cluster.save.call_count) + + def test_poll_and_check_rollback_failed_def_ng(self): + cluster, poller = self.setup_poll_test( + default_stack_status=cluster_status.DELETE_FAILED) + + ng = self._create_nodegroup( + cluster, 'ng', 'stack2', + stack_status=cluster_status.DELETE_IN_PROGRESS) + + cluster.status = cluster_status.DELETE_IN_PROGRESS + poller.poll_and_check() + + for def_ng in self.def_ngs: + self.assertEqual(cluster_status.DELETE_FAILED, def_ng.status) + self.assertEqual(2, def_ng.save.call_count) + + self.assertEqual(cluster_status.DELETE_IN_PROGRESS, ng.status) + self.assertEqual(1, ng.save.call_count) + + self.assertEqual(cluster_status.DELETE_IN_PROGRESS, cluster.status) + self.assertEqual(1, cluster.save.call_count) + + def test_poll_and_check_delete_failed_def_ng(self): + cluster, poller = self.setup_poll_test( + default_stack_status=cluster_status.DELETE_FAILED) + + ng = self._create_nodegroup( + cluster, 'ng', 'stack2', + stack_status=cluster_status.DELETE_COMPLETE) + + cluster.status = cluster_status.DELETE_IN_PROGRESS + poller.poll_and_check() + + for def_ng in self.def_ngs: + self.assertEqual(cluster_status.DELETE_FAILED, def_ng.status) + self.assertEqual(2, def_ng.save.call_count) + + # Check that the non-default ng was deleted + self.assertEqual(1, ng.destroy.call_count) + + self.assertEqual(cluster_status.DELETE_FAILED, cluster.status) + self.assertEqual(1, cluster.save.call_count) + + self.assertIn('worker_ng', cluster.status_reason) + self.assertIn('master_ng', cluster.status_reason)